postgresql-charms-single-kernel 16.3.17__tar.gz → 16.3.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/PKG-INFO +1 -1
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/pyproject.toml +1 -1
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/pyproject.toml.orig +1 -1
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/state.py +21 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/observer.py +53 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/watcher.py +4 -4
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/observer.py +38 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/patroni.py +2 -2
- postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/managers/raft.py +284 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/raft_observer.py +3 -1
- postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/templates/vm/raft-observer.service.j2 +13 -0
- postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/templates/vm/raft-observer.timer.j2 +10 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/LICENSE +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/README.md +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/abstract_charm.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/k8s_charm.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/vm_charm.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/postgresql.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/pyproject.toml +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/enums.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/exceptions.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/literals.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/locales.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/pyproject.toml +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/statuses.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/config.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/peer_relation.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/relation_state.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/s3.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/async_replication.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/backup.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/database.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/ldap.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/logical_replication.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/postgresql.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/tls.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/tls_transfer.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v0/data_interfaces.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v0/s3.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v1/data_models.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/glauth_k8s/v0/ldap.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/async_replication.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/backup.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/base.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/cluster.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/config.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/database.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/k8s.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/logical_replication.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/refresh.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/restore.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/tls.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/authorisation_rules_observer.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/cluster_topology_observer.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/rotate_logs.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/k8s/pgbackrest.conf.j2 +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/k8s/pgbackrest.logrotate.j2 +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/patroni.yml.j2 +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/vm/pgbackrest.conf.j2 +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/vm/pgbackrest.logrotate.j2 +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/arch.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/backup.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/filesystem.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/postgresql.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/s3.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/secret.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/status.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/base.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/k8s.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/__init__.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/base.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/k8s.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/vm.py +0 -0
- {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/vm.py +0 -0
{postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: postgresql-charms-single-kernel
|
|
3
|
-
Version: 16.3.
|
|
3
|
+
Version: 16.3.18
|
|
4
4
|
Summary: Shared and reusable code for PostgreSQL-related charms
|
|
5
5
|
Author: Canonical Data Platform
|
|
6
6
|
Author-email: Canonical Data Platform <data-platform@lists.launchpad.net>
|
|
@@ -579,3 +579,24 @@ class CharmState(Object):
|
|
|
579
579
|
def replicas_endpoint(self) -> str:
|
|
580
580
|
"""Returns the endpoint of the replicas instances' service."""
|
|
581
581
|
return self._build_service_name("replicas")
|
|
582
|
+
|
|
583
|
+
def has_raft_keys(self) -> bool:
|
|
584
|
+
"""Checks for the presence of raft recovery keys in peer data."""
|
|
585
|
+
for key in self.application.data:
|
|
586
|
+
if key.startswith("raft_"):
|
|
587
|
+
return True
|
|
588
|
+
|
|
589
|
+
return any(key.startswith("raft_") for key in self.peer.data)
|
|
590
|
+
|
|
591
|
+
@property
|
|
592
|
+
def units_ips(self) -> set[str]:
|
|
593
|
+
"""Fetch current list of peers IPs."""
|
|
594
|
+
addresses = set()
|
|
595
|
+
|
|
596
|
+
if self.unit_ip:
|
|
597
|
+
addresses.add(self.unit_ip)
|
|
598
|
+
if self.peer_relation:
|
|
599
|
+
for unit in self.peer_relation.units:
|
|
600
|
+
if ip := self.peer_relation.data[unit].get(f"{PEER_RELATION}-address"):
|
|
601
|
+
addresses.add(ip)
|
|
602
|
+
return addresses
|
|
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
|
|
|
9
9
|
from ops import CharmEvents, EventBase, EventSource, Object
|
|
10
10
|
|
|
11
11
|
from single_kernel_postgresql.config.enums import Substrates
|
|
12
|
+
from single_kernel_postgresql.config.literals import RAFT_PARTNER_PREFIX, RAFT_PORT
|
|
12
13
|
|
|
13
14
|
if TYPE_CHECKING:
|
|
14
15
|
from single_kernel_postgresql.charms.abstract_charm import AbstractPostgreSQLCharm
|
|
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
|
|
|
17
18
|
from single_kernel_postgresql.managers.async_replication import AsyncReplicationManager
|
|
18
19
|
from single_kernel_postgresql.managers.database import DatabaseManager
|
|
19
20
|
from single_kernel_postgresql.managers.patroni import PatroniManager
|
|
21
|
+
from single_kernel_postgresql.managers.raft import RaftManager
|
|
20
22
|
from single_kernel_postgresql.managers.tls import TLSManager
|
|
21
23
|
from single_kernel_postgresql.workload.base import BaseWorkload
|
|
22
24
|
|
|
@@ -57,6 +59,7 @@ class ObserverEventsHandler(Object):
|
|
|
57
59
|
async_replication_manager: "AsyncReplicationManager",
|
|
58
60
|
database_manager: "DatabaseManager",
|
|
59
61
|
patroni_manager: "PatroniManager",
|
|
62
|
+
raft_manager: "RaftManager | None",
|
|
60
63
|
tls_manager: "TLSManager",
|
|
61
64
|
watcher_handler: "WatcherEventsHandler",
|
|
62
65
|
):
|
|
@@ -68,12 +71,14 @@ class ObserverEventsHandler(Object):
|
|
|
68
71
|
self.async_replication_manager = async_replication_manager
|
|
69
72
|
self.database_manager = database_manager
|
|
70
73
|
self.patroni_manager = patroni_manager
|
|
74
|
+
self.raft_manager = raft_manager
|
|
71
75
|
self.tls_manager = tls_manager
|
|
72
76
|
|
|
73
77
|
if self.state.substrate == Substrates.VM:
|
|
74
78
|
self.framework.observe(
|
|
75
79
|
self.charm.on.cluster_topology_change, self._on_cluster_topology_change
|
|
76
80
|
)
|
|
81
|
+
self.framework.observe(self.charm.on.raft_reconnect, self._on_raft_reconnect)
|
|
77
82
|
self.framework.observe(self.charm.on.databases_change, self._on_databases_change)
|
|
78
83
|
|
|
79
84
|
def _on_databases_change(self, _):
|
|
@@ -96,3 +101,51 @@ class ObserverEventsHandler(Object):
|
|
|
96
101
|
)
|
|
97
102
|
self.charm.set_primary_status_message()
|
|
98
103
|
self.async_replication_manager.update_async_replication_data()
|
|
104
|
+
|
|
105
|
+
def _on_raft_reconnect(self, _) -> None:
|
|
106
|
+
if not self.raft_manager:
|
|
107
|
+
raise ValueError("No RAFT manager set.")
|
|
108
|
+
|
|
109
|
+
raft_status = self.raft_manager.get_raft_status()
|
|
110
|
+
logger.debug(f"Local raft status: {raft_status}")
|
|
111
|
+
if (
|
|
112
|
+
not raft_status
|
|
113
|
+
or not self.state.unit_ip
|
|
114
|
+
or not self.state.application.is_cluster_initialised
|
|
115
|
+
or self.state.unit_ip not in self.state.application.members_ips
|
|
116
|
+
or self.state.has_raft_keys()
|
|
117
|
+
or (
|
|
118
|
+
not self.state.application.members_ips
|
|
119
|
+
and not self.watcher_handler.watcher_raft_address
|
|
120
|
+
)
|
|
121
|
+
):
|
|
122
|
+
return
|
|
123
|
+
|
|
124
|
+
if all(
|
|
125
|
+
raft_status[partner] == 2
|
|
126
|
+
for partner in raft_status
|
|
127
|
+
if partner.startswith(RAFT_PARTNER_PREFIX)
|
|
128
|
+
):
|
|
129
|
+
logger.debug("All raft members are active.")
|
|
130
|
+
return
|
|
131
|
+
|
|
132
|
+
logger.info("Potentially stuck Raft connection detected. Re-adding Raft member.")
|
|
133
|
+
local_addr = f"{self.state.unit_ip}:{RAFT_PORT}"
|
|
134
|
+
remote_addr = (
|
|
135
|
+
watcher_addr
|
|
136
|
+
if (watcher_addr := self.watcher_handler.watcher_raft_address)
|
|
137
|
+
and self.watcher_handler.is_active
|
|
138
|
+
else f"{next(member for member in self.state.application.members_ips if member != self.state.unit_ip)}:{RAFT_PORT}"
|
|
139
|
+
)
|
|
140
|
+
try:
|
|
141
|
+
self.raft_manager.remove_raft_member(
|
|
142
|
+
local_addr, remote_address=remote_addr, set_raft_flags=False
|
|
143
|
+
)
|
|
144
|
+
except Exception:
|
|
145
|
+
logger.exception("Unable to remove Raft member")
|
|
146
|
+
return
|
|
147
|
+
try:
|
|
148
|
+
self.raft_manager.add_raft_member(local_addr, remote_address=remote_addr)
|
|
149
|
+
except Exception:
|
|
150
|
+
logger.exception("Unable to add Raft member")
|
|
151
|
+
return
|
|
@@ -126,7 +126,7 @@ class WatcherEventsHandler(Object):
|
|
|
126
126
|
self._relation.data[self.state.peer.unit.app].update({"disable-watcher": "True"})
|
|
127
127
|
try:
|
|
128
128
|
if self.watcher_raft_address:
|
|
129
|
-
self.charm.
|
|
129
|
+
self.charm.raft_manager.remove_raft_member(self.watcher_raft_address)
|
|
130
130
|
except Exception as e:
|
|
131
131
|
logger.warning(f"Error remove Raft watcher: {e}")
|
|
132
132
|
|
|
@@ -232,7 +232,7 @@ class WatcherEventsHandler(Object):
|
|
|
232
232
|
# Only the leader handles Raft membership changes and user management
|
|
233
233
|
# to avoid race conditions between multiple PostgreSQL units
|
|
234
234
|
if self.state.peer.is_app_leader:
|
|
235
|
-
self.charm.
|
|
235
|
+
self.charm.raft_manager.cleanup_raft_cluster()
|
|
236
236
|
self._ensure_watcher_user()
|
|
237
237
|
# Update Patroni configuration to include watcher in Raft
|
|
238
238
|
self.charm.update_config()
|
|
@@ -255,7 +255,7 @@ class WatcherEventsHandler(Object):
|
|
|
255
255
|
logger.info("Watcher relation broken, updating Patroni configuration")
|
|
256
256
|
self.watcher_raft_address = None
|
|
257
257
|
if self.state.peer.is_app_leader:
|
|
258
|
-
self.charm.
|
|
258
|
+
self.charm.raft_manager.cleanup_raft_cluster()
|
|
259
259
|
# Update Patroni configuration without the watcher
|
|
260
260
|
self.charm.update_config()
|
|
261
261
|
|
|
@@ -424,7 +424,7 @@ class WatcherEventsHandler(Object):
|
|
|
424
424
|
# Collect PostgreSQL unit endpoints using fresh IPs from unit relation data.
|
|
425
425
|
# _units_ips reads directly from unit relation data (always fresh), while
|
|
426
426
|
# _peer_members_ips reads from app peer data (may be stale after network disruptions).
|
|
427
|
-
pg_endpoints: list[str] = sorted(self.
|
|
427
|
+
pg_endpoints: list[str] = sorted(self.state.units_ips)
|
|
428
428
|
if not pg_endpoints:
|
|
429
429
|
logger.warning("No PostgreSQL endpoints available")
|
|
430
430
|
return
|
|
@@ -7,10 +7,15 @@ import logging
|
|
|
7
7
|
import os
|
|
8
8
|
import signal
|
|
9
9
|
import subprocess
|
|
10
|
+
from contextlib import suppress
|
|
11
|
+
from importlib import resources
|
|
10
12
|
from pathlib import Path
|
|
11
13
|
from sys import version_info
|
|
12
14
|
from typing import TYPE_CHECKING
|
|
13
15
|
|
|
16
|
+
with suppress(ImportError):
|
|
17
|
+
from charmlibs.systemd import daemon_reload, service_enable
|
|
18
|
+
from jinja2 import Template
|
|
14
19
|
from ops import ActiveStatus
|
|
15
20
|
|
|
16
21
|
from single_kernel_postgresql.config.enums import Substrates
|
|
@@ -20,6 +25,7 @@ from single_kernel_postgresql.config.literals import (
|
|
|
20
25
|
OBSERVER_VM_LOG_FILE_PATH,
|
|
21
26
|
)
|
|
22
27
|
from single_kernel_postgresql.managers.base import BaseManager
|
|
28
|
+
from single_kernel_postgresql.utils import render_file
|
|
23
29
|
|
|
24
30
|
if TYPE_CHECKING:
|
|
25
31
|
from single_kernel_postgresql.core.state import CharmState
|
|
@@ -129,6 +135,38 @@ class ObserverManager(BaseManager):
|
|
|
129
135
|
except OSError:
|
|
130
136
|
pass
|
|
131
137
|
|
|
138
|
+
def start_raft_observer(self) -> None:
|
|
139
|
+
"""Render systemd units and start the observer."""
|
|
140
|
+
timer_service_file = "/etc/systemd/system/raft-observer.timer"
|
|
141
|
+
oneshot_service_file = "/etc/systemd/system/raft-observer.service"
|
|
142
|
+
|
|
143
|
+
template = Template(
|
|
144
|
+
resources
|
|
145
|
+
.files("single_kernel_postgresql.templates.vm")
|
|
146
|
+
.joinpath("raft-observer.service.j2")
|
|
147
|
+
.read_text()
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
rendered = template.render(
|
|
151
|
+
envvars=copy_environment(),
|
|
152
|
+
script="-m single_kernel_postgresql.scripts.raft_observer",
|
|
153
|
+
)
|
|
154
|
+
render_file(Substrates.VM, oneshot_service_file, rendered, 0o644, change_owner=False)
|
|
155
|
+
|
|
156
|
+
template = Template(
|
|
157
|
+
resources
|
|
158
|
+
.files("single_kernel_postgresql.templates.vm")
|
|
159
|
+
.joinpath("raft-observer.timer.j2")
|
|
160
|
+
.read_text()
|
|
161
|
+
)
|
|
162
|
+
rendered = template.render()
|
|
163
|
+
render_file(Substrates.VM, timer_service_file, rendered, 0o644, change_owner=False)
|
|
164
|
+
|
|
165
|
+
# Reload systemd to pick up the new service
|
|
166
|
+
daemon_reload()
|
|
167
|
+
service_enable(timer_service_file, "--now")
|
|
168
|
+
logger.info("Installed and enabled raft observer timer")
|
|
169
|
+
|
|
132
170
|
|
|
133
171
|
def copy_environment() -> dict[str, str]:
|
|
134
172
|
"""Environment variables to pass to a script."""
|
|
@@ -206,8 +206,8 @@ class PatroniManager(BaseManager):
|
|
|
206
206
|
endpoints = []
|
|
207
207
|
if self.state.endpoint:
|
|
208
208
|
endpoints.append(self.state.endpoint)
|
|
209
|
-
|
|
210
|
-
|
|
209
|
+
for peer in self.state.endpoints:
|
|
210
|
+
endpoints.append(peer)
|
|
211
211
|
# Request info from cluster endpoint (which returns all members of the cluster).
|
|
212
212
|
if response := parallel_patroni_get_request(
|
|
213
213
|
f"/{PATRONI_CLUSTER_STATUS_ENDPOINT}",
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# Copyright 2021 Canonical Ltd.
|
|
3
|
+
# See LICENSE file for licensing details.
|
|
4
|
+
|
|
5
|
+
"""Raft manager for VM substrate."""
|
|
6
|
+
|
|
7
|
+
import logging
|
|
8
|
+
import shutil
|
|
9
|
+
from contextlib import suppress
|
|
10
|
+
from typing import TYPE_CHECKING
|
|
11
|
+
|
|
12
|
+
with suppress(ImportError):
|
|
13
|
+
import psutil
|
|
14
|
+
from pysyncobj.utility import TcpUtility, UtilityException
|
|
15
|
+
from ops import BlockedStatus
|
|
16
|
+
from tenacity import Retrying, wait_fixed
|
|
17
|
+
|
|
18
|
+
from single_kernel_postgresql.config.exceptions import (
|
|
19
|
+
AddRaftMemberFailedError,
|
|
20
|
+
RaftNotPromotedError,
|
|
21
|
+
RaftPostgresqlNotUpError,
|
|
22
|
+
RaftPostgresqlStillUpError,
|
|
23
|
+
RemoveRaftMemberFailedError,
|
|
24
|
+
)
|
|
25
|
+
from single_kernel_postgresql.config.literals import PEER_RELATION, RAFT_PARTNER_PREFIX, RAFT_PORT
|
|
26
|
+
from single_kernel_postgresql.managers.base import BaseManager
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
from collections.abc import Callable
|
|
30
|
+
|
|
31
|
+
from ops import StatusBase
|
|
32
|
+
|
|
33
|
+
from single_kernel_postgresql.core.state import CharmState
|
|
34
|
+
from single_kernel_postgresql.events.watcher import WatcherEventsHandler
|
|
35
|
+
from single_kernel_postgresql.managers.patroni import PatroniManager
|
|
36
|
+
from single_kernel_postgresql.workload.base import BaseWorkload
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class RaftManager(BaseManager):
|
|
42
|
+
"""RAFT manager for VM substrate."""
|
|
43
|
+
|
|
44
|
+
def __init__(
|
|
45
|
+
self,
|
|
46
|
+
state: "CharmState",
|
|
47
|
+
workload: "BaseWorkload",
|
|
48
|
+
patroni_manager: "PatroniManager",
|
|
49
|
+
# TODO split off into a manager
|
|
50
|
+
watcher_handler: "WatcherEventsHandler",
|
|
51
|
+
update_config,
|
|
52
|
+
set_unit_status: "Callable[[StatusBase], None]",
|
|
53
|
+
peer_relation_changed,
|
|
54
|
+
remove_from_members_ips,
|
|
55
|
+
) -> None:
|
|
56
|
+
super().__init__(state, workload, "raft_manager")
|
|
57
|
+
self.state = state
|
|
58
|
+
self.workload = workload
|
|
59
|
+
self.patroni_manager = patroni_manager
|
|
60
|
+
self.watcher_handler = watcher_handler
|
|
61
|
+
|
|
62
|
+
# Charm fucts
|
|
63
|
+
self.update_config = update_config
|
|
64
|
+
self.set_unit_status = set_unit_status
|
|
65
|
+
self.peer_relation_changed = peer_relation_changed
|
|
66
|
+
self.remove_from_members_ips = remove_from_members_ips
|
|
67
|
+
|
|
68
|
+
def has_raft_quorum(self) -> bool:
|
|
69
|
+
"""Check if raft cluster has quorum."""
|
|
70
|
+
# Get the status of the raft cluster.
|
|
71
|
+
syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
|
|
72
|
+
|
|
73
|
+
raft_host = "127.0.0.1:2222"
|
|
74
|
+
try:
|
|
75
|
+
raft_status = syncobj_util.executeCommand(raft_host, ["status"])
|
|
76
|
+
except UtilityException:
|
|
77
|
+
logger.warning("Has raft quorum: Cannot connect to raft cluster")
|
|
78
|
+
return False
|
|
79
|
+
if not raft_status:
|
|
80
|
+
logger.warning("Has raft quorum: No status reported")
|
|
81
|
+
return False
|
|
82
|
+
return raft_status["has_quorum"]
|
|
83
|
+
|
|
84
|
+
def remove_raft_data(self) -> None:
|
|
85
|
+
"""Stops Patroni and removes the raft journals."""
|
|
86
|
+
logger.info("Stopping patroni")
|
|
87
|
+
self.patroni_manager.stop_patroni()
|
|
88
|
+
|
|
89
|
+
logger.info("Wait for postgresql to stop")
|
|
90
|
+
for attempt in Retrying(wait=wait_fixed(5)):
|
|
91
|
+
with attempt:
|
|
92
|
+
for proc in psutil.process_iter(["name"]):
|
|
93
|
+
if proc.name() == "postgres":
|
|
94
|
+
raise RaftPostgresqlStillUpError()
|
|
95
|
+
|
|
96
|
+
logger.info("Removing raft data")
|
|
97
|
+
try:
|
|
98
|
+
path = self.workload.paths.patroni_conf / "raft"
|
|
99
|
+
if path.exists() and path.is_dir():
|
|
100
|
+
shutil.rmtree(str(path))
|
|
101
|
+
except OSError as e:
|
|
102
|
+
raise Exception(
|
|
103
|
+
f"Failed to remove previous cluster information with error: {e!s}"
|
|
104
|
+
) from e
|
|
105
|
+
logger.info("Raft ready to reinitialise")
|
|
106
|
+
|
|
107
|
+
def reinitialise_raft_data(self) -> None:
|
|
108
|
+
"""Reinitialise the raft journals and promoting the unit to leader. Should only be run on sync replicas."""
|
|
109
|
+
logger.info("Rerendering patroni config without peers")
|
|
110
|
+
# TODO factor out
|
|
111
|
+
self.update_config(no_peers=True)
|
|
112
|
+
logger.info("Starting patroni")
|
|
113
|
+
self.patroni_manager.start_patroni()
|
|
114
|
+
|
|
115
|
+
logger.info("Waiting for new raft cluster to initialise")
|
|
116
|
+
for attempt in Retrying(wait=wait_fixed(5)):
|
|
117
|
+
with attempt:
|
|
118
|
+
health_status = self.patroni_manager.get_patroni_health()
|
|
119
|
+
if (
|
|
120
|
+
health_status["role"] not in ["leader", "master"]
|
|
121
|
+
or health_status["state"] != "running"
|
|
122
|
+
):
|
|
123
|
+
raise RaftNotPromotedError()
|
|
124
|
+
|
|
125
|
+
logger.info("Restarting patroni")
|
|
126
|
+
self.patroni_manager.restart_patroni()
|
|
127
|
+
for attempt in Retrying(wait=wait_fixed(5)):
|
|
128
|
+
with attempt:
|
|
129
|
+
found_postgres = False
|
|
130
|
+
for proc in psutil.process_iter(["name"]):
|
|
131
|
+
if proc.name() == "postgres":
|
|
132
|
+
found_postgres = True
|
|
133
|
+
break
|
|
134
|
+
if not found_postgres:
|
|
135
|
+
raise RaftPostgresqlNotUpError()
|
|
136
|
+
logger.info("Raft should be unstuck")
|
|
137
|
+
|
|
138
|
+
def cleanup_raft_cluster(self) -> bool:
|
|
139
|
+
"""Cleanup RAFT members not belonging to the current cluster or not a related watcher."""
|
|
140
|
+
# Get Raft cluster status to find all members
|
|
141
|
+
try:
|
|
142
|
+
if not self.workload.is_patroni_running():
|
|
143
|
+
logger.warning("Raft cleanup: Patroni service not running.")
|
|
144
|
+
return True
|
|
145
|
+
syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
|
|
146
|
+
if raft_status := syncobj_util.executeCommand(f"127.0.0.1:{RAFT_PORT}", ["status"]):
|
|
147
|
+
# Find all partner nodes in the Raft cluster
|
|
148
|
+
# Keys look like: partner_node_status_server_10.131.50.142:2222
|
|
149
|
+
for key in raft_status:
|
|
150
|
+
if key.startswith(RAFT_PARTNER_PREFIX) and raft_status[key] != 2:
|
|
151
|
+
member_addr = key.replace(RAFT_PARTNER_PREFIX, "")
|
|
152
|
+
member_ip = member_addr.split(":")[0]
|
|
153
|
+
|
|
154
|
+
# Check if this is a stale watcher (not a PostgreSQL node and not current watcher)
|
|
155
|
+
if (
|
|
156
|
+
member_ip not in self.state.units_ips
|
|
157
|
+
and member_addr != self.watcher_handler.watcher_raft_address
|
|
158
|
+
):
|
|
159
|
+
logger.info(f"Removing stale Raft member: {member_addr}")
|
|
160
|
+
self.remove_raft_member(member_addr)
|
|
161
|
+
# TODO Factor out
|
|
162
|
+
self.remove_from_members_ips(member_ip)
|
|
163
|
+
return True
|
|
164
|
+
return False
|
|
165
|
+
except Exception as e:
|
|
166
|
+
logger.debug(f"Error during Raft cleanup: {e}")
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
def _set_stuck_raft_flag(self) -> None:
|
|
170
|
+
# TODO factor out
|
|
171
|
+
self.set_unit_status(BlockedStatus("Raft majority loss, run: promote-to-primary"))
|
|
172
|
+
logger.warning("Remove raft member: Stuck raft cluster detected")
|
|
173
|
+
self.state.peer.data.update({"raft_stuck": "True"})
|
|
174
|
+
|
|
175
|
+
# Leader doesn't always trigger when changing it's own peer data.
|
|
176
|
+
if self.state.peer.is_app_leader:
|
|
177
|
+
# TODO Factor out
|
|
178
|
+
self.peer_relation_changed.emit(
|
|
179
|
+
unit=self.state.peer.unit,
|
|
180
|
+
app=self.state.peer.unit.app,
|
|
181
|
+
relation=self.state.model.get_relation(PEER_RELATION),
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
def get_raft_status(self) -> dict | None:
|
|
185
|
+
"""Get local raft status."""
|
|
186
|
+
# Get the status of the raft cluster.
|
|
187
|
+
syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
|
|
188
|
+
|
|
189
|
+
raft_host = f"127.0.0.1:{RAFT_PORT}"
|
|
190
|
+
with suppress(UtilityException):
|
|
191
|
+
return syncobj_util.executeCommand(raft_host, ["status"])
|
|
192
|
+
|
|
193
|
+
def remove_raft_member(
|
|
194
|
+
self, member_address: str, remote_address: str | None = None, set_raft_flags: bool = True
|
|
195
|
+
) -> None:
|
|
196
|
+
"""Remove a member from the raft cluster.
|
|
197
|
+
|
|
198
|
+
The raft cluster is a different cluster from the Patroni cluster.
|
|
199
|
+
It is responsible for defining which Patroni member can update
|
|
200
|
+
the primary member in the DCS.
|
|
201
|
+
|
|
202
|
+
Raises:
|
|
203
|
+
RaftMemberNotFoundError: if the member to be removed
|
|
204
|
+
is not part of the raft cluster.
|
|
205
|
+
"""
|
|
206
|
+
if self.state.has_raft_keys():
|
|
207
|
+
logger.debug("Remove raft member: Raft already in recovery")
|
|
208
|
+
return
|
|
209
|
+
|
|
210
|
+
# Get the status of the raft cluster.
|
|
211
|
+
syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
|
|
212
|
+
|
|
213
|
+
raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}"
|
|
214
|
+
try:
|
|
215
|
+
raft_status = syncobj_util.executeCommand(raft_host, ["status"])
|
|
216
|
+
except UtilityException as e:
|
|
217
|
+
logger.warning("Remove raft member: Cannot connect to raft cluster")
|
|
218
|
+
raise RemoveRaftMemberFailedError() from e
|
|
219
|
+
if not raft_status:
|
|
220
|
+
logger.warning("Remove raft member: No raft status")
|
|
221
|
+
raise RemoveRaftMemberFailedError() from None
|
|
222
|
+
|
|
223
|
+
# Check whether the member is still part of the raft cluster.
|
|
224
|
+
if f"{RAFT_PARTNER_PREFIX}{member_address}" not in raft_status:
|
|
225
|
+
logger.debug("Remove raft member: Address already removed")
|
|
226
|
+
return
|
|
227
|
+
|
|
228
|
+
# If there's no quorum and the leader left raft cluster is stuck
|
|
229
|
+
if raft_status["has_quorum"] and not raft_status["leader"]:
|
|
230
|
+
logger.warning("Remove raft member: No raft leader")
|
|
231
|
+
raise RemoveRaftMemberFailedError() from None
|
|
232
|
+
if (
|
|
233
|
+
not raft_status["has_quorum"]
|
|
234
|
+
and (not raft_status["leader"] or raft_status["leader"].address == member_address)
|
|
235
|
+
and set_raft_flags
|
|
236
|
+
):
|
|
237
|
+
self._set_stuck_raft_flag()
|
|
238
|
+
return
|
|
239
|
+
|
|
240
|
+
# Remove the member from the raft cluster.
|
|
241
|
+
try:
|
|
242
|
+
result = syncobj_util.executeCommand(raft_host, ["remove", member_address])
|
|
243
|
+
except UtilityException as e:
|
|
244
|
+
logger.debug("Remove raft member: Remove call failed")
|
|
245
|
+
raise RemoveRaftMemberFailedError() from e
|
|
246
|
+
|
|
247
|
+
if not result or not result.startswith("SUCCESS"):
|
|
248
|
+
logger.debug(f"Remove raft member: Remove call not successful with {result}")
|
|
249
|
+
raise RemoveRaftMemberFailedError() from None
|
|
250
|
+
|
|
251
|
+
def add_raft_member(self, member_address: str, remote_address: str | None = None) -> None:
|
|
252
|
+
"""Add a member to the raft cluster."""
|
|
253
|
+
if self.state.has_raft_keys():
|
|
254
|
+
logger.debug("Add raft member: Raft already in recovery")
|
|
255
|
+
return
|
|
256
|
+
|
|
257
|
+
# Get the status of the raft cluster.
|
|
258
|
+
syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
|
|
259
|
+
|
|
260
|
+
raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}"
|
|
261
|
+
try:
|
|
262
|
+
raft_status = syncobj_util.executeCommand(raft_host, ["status"])
|
|
263
|
+
except UtilityException as e:
|
|
264
|
+
logger.warning("Add raft member: Cannot connect to raft cluster")
|
|
265
|
+
raise AddRaftMemberFailedError() from e
|
|
266
|
+
|
|
267
|
+
if not raft_status:
|
|
268
|
+
logger.warning("Add raft member: No raft status")
|
|
269
|
+
raise AddRaftMemberFailedError() from None
|
|
270
|
+
|
|
271
|
+
# Check whether the member is still part of the raft cluster.
|
|
272
|
+
if f"{RAFT_PARTNER_PREFIX}{member_address}" in raft_status:
|
|
273
|
+
logger.debug("Add raft member: Address already added")
|
|
274
|
+
return
|
|
275
|
+
|
|
276
|
+
try:
|
|
277
|
+
result = syncobj_util.executeCommand(raft_host, ["add", member_address])
|
|
278
|
+
except UtilityException as e:
|
|
279
|
+
logger.debug("Add raft member: Remove call failed")
|
|
280
|
+
raise AddRaftMemberFailedError() from e
|
|
281
|
+
|
|
282
|
+
if not result or not result.startswith("SUCCESS"):
|
|
283
|
+
logger.debug(f"Add raft member: Remove call not successful with {result}")
|
|
284
|
+
raise AddRaftMemberFailedError() from None
|
|
@@ -11,12 +11,14 @@ from pysyncobj.utility import TcpUtility
|
|
|
11
11
|
from yaml import safe_load
|
|
12
12
|
|
|
13
13
|
from single_kernel_postgresql.config.literals import (
|
|
14
|
+
BASE_SNAP_DIR,
|
|
14
15
|
PATRONI_CONF_PATH,
|
|
15
16
|
RAFT_PARTNER_PREFIX,
|
|
16
17
|
RAFT_PORT,
|
|
18
|
+
SNAP_COMMON,
|
|
17
19
|
)
|
|
18
20
|
|
|
19
|
-
PATRONI_CONF_FILE_PATH = f"{PATRONI_CONF_PATH}/patroni.yaml"
|
|
21
|
+
PATRONI_CONF_FILE_PATH = f"{BASE_SNAP_DIR}/{SNAP_COMMON}{PATRONI_CONF_PATH}/patroni.yaml"
|
|
20
22
|
|
|
21
23
|
|
|
22
24
|
def dispatch(custom_event):
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
[Unit]
|
|
2
|
+
Description=Patroni Raft observer service
|
|
3
|
+
After=snap.charmed-postgresql.patroni.service
|
|
4
|
+
Wants=network.target
|
|
5
|
+
|
|
6
|
+
[Service]
|
|
7
|
+
Type=oneshot
|
|
8
|
+
{% for var, val in envvars.items() -%}
|
|
9
|
+
Environment="{{ var }}={{ val }}"
|
|
10
|
+
{% endfor -%}
|
|
11
|
+
ExecStart=python3 {{ script }}
|
|
12
|
+
StandardOutput=journal
|
|
13
|
+
StandardError=journal
|
|
File without changes
|
{postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/README.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|