postgresql-charms-single-kernel 16.3.17__tar.gz → 16.3.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/PKG-INFO +1 -1
  2. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/pyproject.toml +1 -1
  3. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/pyproject.toml.orig +1 -1
  4. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/state.py +21 -0
  5. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/observer.py +53 -0
  6. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/watcher.py +4 -4
  7. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/observer.py +38 -0
  8. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/patroni.py +2 -2
  9. postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/managers/raft.py +284 -0
  10. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/raft_observer.py +3 -1
  11. postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/templates/vm/raft-observer.service.j2 +13 -0
  12. postgresql_charms_single_kernel-16.3.18/single_kernel_postgresql/templates/vm/raft-observer.timer.j2 +10 -0
  13. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/LICENSE +0 -0
  14. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/README.md +0 -0
  15. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/__init__.py +0 -0
  16. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/__init__.py +0 -0
  17. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/abstract_charm.py +0 -0
  18. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/k8s_charm.py +0 -0
  19. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/charms/vm_charm.py +0 -0
  20. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/__init__.py +0 -0
  21. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/postgresql.py +0 -0
  22. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/compat/pyproject.toml +0 -0
  23. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/__init__.py +0 -0
  24. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/enums.py +0 -0
  25. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/exceptions.py +0 -0
  26. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/literals.py +0 -0
  27. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/locales.py +0 -0
  28. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/pyproject.toml +0 -0
  29. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/config/statuses.py +0 -0
  30. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/__init__.py +0 -0
  31. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/config.py +0 -0
  32. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/peer_relation.py +0 -0
  33. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/relation_state.py +0 -0
  34. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/core/s3.py +0 -0
  35. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/__init__.py +0 -0
  36. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/async_replication.py +0 -0
  37. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/backup.py +0 -0
  38. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/database.py +0 -0
  39. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/ldap.py +0 -0
  40. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/logical_replication.py +0 -0
  41. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/postgresql.py +0 -0
  42. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/tls.py +0 -0
  43. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/events/tls_transfer.py +0 -0
  44. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v0/data_interfaces.py +0 -0
  45. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v0/s3.py +0 -0
  46. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/data_platform_libs/v1/data_models.py +0 -0
  47. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/lib/charms/glauth_k8s/v0/ldap.py +0 -0
  48. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/__init__.py +0 -0
  49. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/async_replication.py +0 -0
  50. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/backup.py +0 -0
  51. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/base.py +0 -0
  52. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/cluster.py +0 -0
  53. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/config.py +0 -0
  54. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/database.py +0 -0
  55. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/k8s.py +0 -0
  56. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/logical_replication.py +0 -0
  57. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/refresh.py +0 -0
  58. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/restore.py +0 -0
  59. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/managers/tls.py +0 -0
  60. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/authorisation_rules_observer.py +0 -0
  61. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/cluster_topology_observer.py +0 -0
  62. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/scripts/rotate_logs.py +0 -0
  63. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/__init__.py +0 -0
  64. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/k8s/pgbackrest.conf.j2 +0 -0
  65. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/k8s/pgbackrest.logrotate.j2 +0 -0
  66. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/patroni.yml.j2 +0 -0
  67. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/vm/pgbackrest.conf.j2 +0 -0
  68. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/templates/vm/pgbackrest.logrotate.j2 +0 -0
  69. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/__init__.py +0 -0
  70. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/arch.py +0 -0
  71. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/backup.py +0 -0
  72. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/filesystem.py +0 -0
  73. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/postgresql.py +0 -0
  74. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/s3.py +0 -0
  75. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/secret.py +0 -0
  76. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/utils/status.py +0 -0
  77. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/__init__.py +0 -0
  78. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/base.py +0 -0
  79. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/k8s.py +0 -0
  80. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/__init__.py +0 -0
  81. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/base.py +0 -0
  82. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/k8s.py +0 -0
  83. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/paths/vm.py +0 -0
  84. {postgresql_charms_single_kernel-16.3.17 → postgresql_charms_single_kernel-16.3.18}/single_kernel_postgresql/workload/vm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: postgresql-charms-single-kernel
3
- Version: 16.3.17
3
+ Version: 16.3.18
4
4
  Summary: Shared and reusable code for PostgreSQL-related charms
5
5
  Author: Canonical Data Platform
6
6
  Author-email: Canonical Data Platform <data-platform@lists.launchpad.net>
@@ -1,7 +1,7 @@
1
1
  [project]
2
2
  name = "postgresql-charms-single-kernel"
3
3
  description = "Shared and reusable code for PostgreSQL-related charms"
4
- version = "16.3.17"
4
+ version = "16.3.18"
5
5
  readme = "README.md"
6
6
  classifiers = [
7
7
  "Development Status :: 3 - Alpha",
@@ -4,7 +4,7 @@
4
4
  [project]
5
5
  name = "postgresql-charms-single-kernel"
6
6
  description = "Shared and reusable code for PostgreSQL-related charms"
7
- version = "16.3.17"
7
+ version = "16.3.18"
8
8
  readme = "README.md"
9
9
  license = {file = "LICENSE"}
10
10
  authors = [
@@ -579,3 +579,24 @@ class CharmState(Object):
579
579
  def replicas_endpoint(self) -> str:
580
580
  """Returns the endpoint of the replicas instances' service."""
581
581
  return self._build_service_name("replicas")
582
+
583
+ def has_raft_keys(self) -> bool:
584
+ """Checks for the presence of raft recovery keys in peer data."""
585
+ for key in self.application.data:
586
+ if key.startswith("raft_"):
587
+ return True
588
+
589
+ return any(key.startswith("raft_") for key in self.peer.data)
590
+
591
+ @property
592
+ def units_ips(self) -> set[str]:
593
+ """Fetch current list of peers IPs."""
594
+ addresses = set()
595
+
596
+ if self.unit_ip:
597
+ addresses.add(self.unit_ip)
598
+ if self.peer_relation:
599
+ for unit in self.peer_relation.units:
600
+ if ip := self.peer_relation.data[unit].get(f"{PEER_RELATION}-address"):
601
+ addresses.add(ip)
602
+ return addresses
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING
9
9
  from ops import CharmEvents, EventBase, EventSource, Object
10
10
 
11
11
  from single_kernel_postgresql.config.enums import Substrates
12
+ from single_kernel_postgresql.config.literals import RAFT_PARTNER_PREFIX, RAFT_PORT
12
13
 
13
14
  if TYPE_CHECKING:
14
15
  from single_kernel_postgresql.charms.abstract_charm import AbstractPostgreSQLCharm
@@ -17,6 +18,7 @@ if TYPE_CHECKING:
17
18
  from single_kernel_postgresql.managers.async_replication import AsyncReplicationManager
18
19
  from single_kernel_postgresql.managers.database import DatabaseManager
19
20
  from single_kernel_postgresql.managers.patroni import PatroniManager
21
+ from single_kernel_postgresql.managers.raft import RaftManager
20
22
  from single_kernel_postgresql.managers.tls import TLSManager
21
23
  from single_kernel_postgresql.workload.base import BaseWorkload
22
24
 
@@ -57,6 +59,7 @@ class ObserverEventsHandler(Object):
57
59
  async_replication_manager: "AsyncReplicationManager",
58
60
  database_manager: "DatabaseManager",
59
61
  patroni_manager: "PatroniManager",
62
+ raft_manager: "RaftManager | None",
60
63
  tls_manager: "TLSManager",
61
64
  watcher_handler: "WatcherEventsHandler",
62
65
  ):
@@ -68,12 +71,14 @@ class ObserverEventsHandler(Object):
68
71
  self.async_replication_manager = async_replication_manager
69
72
  self.database_manager = database_manager
70
73
  self.patroni_manager = patroni_manager
74
+ self.raft_manager = raft_manager
71
75
  self.tls_manager = tls_manager
72
76
 
73
77
  if self.state.substrate == Substrates.VM:
74
78
  self.framework.observe(
75
79
  self.charm.on.cluster_topology_change, self._on_cluster_topology_change
76
80
  )
81
+ self.framework.observe(self.charm.on.raft_reconnect, self._on_raft_reconnect)
77
82
  self.framework.observe(self.charm.on.databases_change, self._on_databases_change)
78
83
 
79
84
  def _on_databases_change(self, _):
@@ -96,3 +101,51 @@ class ObserverEventsHandler(Object):
96
101
  )
97
102
  self.charm.set_primary_status_message()
98
103
  self.async_replication_manager.update_async_replication_data()
104
+
105
+ def _on_raft_reconnect(self, _) -> None:
106
+ if not self.raft_manager:
107
+ raise ValueError("No RAFT manager set.")
108
+
109
+ raft_status = self.raft_manager.get_raft_status()
110
+ logger.debug(f"Local raft status: {raft_status}")
111
+ if (
112
+ not raft_status
113
+ or not self.state.unit_ip
114
+ or not self.state.application.is_cluster_initialised
115
+ or self.state.unit_ip not in self.state.application.members_ips
116
+ or self.state.has_raft_keys()
117
+ or (
118
+ not self.state.application.members_ips
119
+ and not self.watcher_handler.watcher_raft_address
120
+ )
121
+ ):
122
+ return
123
+
124
+ if all(
125
+ raft_status[partner] == 2
126
+ for partner in raft_status
127
+ if partner.startswith(RAFT_PARTNER_PREFIX)
128
+ ):
129
+ logger.debug("All raft members are active.")
130
+ return
131
+
132
+ logger.info("Potentially stuck Raft connection detected. Re-adding Raft member.")
133
+ local_addr = f"{self.state.unit_ip}:{RAFT_PORT}"
134
+ remote_addr = (
135
+ watcher_addr
136
+ if (watcher_addr := self.watcher_handler.watcher_raft_address)
137
+ and self.watcher_handler.is_active
138
+ else f"{next(member for member in self.state.application.members_ips if member != self.state.unit_ip)}:{RAFT_PORT}"
139
+ )
140
+ try:
141
+ self.raft_manager.remove_raft_member(
142
+ local_addr, remote_address=remote_addr, set_raft_flags=False
143
+ )
144
+ except Exception:
145
+ logger.exception("Unable to remove Raft member")
146
+ return
147
+ try:
148
+ self.raft_manager.add_raft_member(local_addr, remote_address=remote_addr)
149
+ except Exception:
150
+ logger.exception("Unable to add Raft member")
151
+ return
@@ -126,7 +126,7 @@ class WatcherEventsHandler(Object):
126
126
  self._relation.data[self.state.peer.unit.app].update({"disable-watcher": "True"})
127
127
  try:
128
128
  if self.watcher_raft_address:
129
- self.charm._patroni.remove_raft_member(self.watcher_raft_address)
129
+ self.charm.raft_manager.remove_raft_member(self.watcher_raft_address)
130
130
  except Exception as e:
131
131
  logger.warning(f"Error remove Raft watcher: {e}")
132
132
 
@@ -232,7 +232,7 @@ class WatcherEventsHandler(Object):
232
232
  # Only the leader handles Raft membership changes and user management
233
233
  # to avoid race conditions between multiple PostgreSQL units
234
234
  if self.state.peer.is_app_leader:
235
- self.charm._patroni.cleanup_raft_cluster()
235
+ self.charm.raft_manager.cleanup_raft_cluster()
236
236
  self._ensure_watcher_user()
237
237
  # Update Patroni configuration to include watcher in Raft
238
238
  self.charm.update_config()
@@ -255,7 +255,7 @@ class WatcherEventsHandler(Object):
255
255
  logger.info("Watcher relation broken, updating Patroni configuration")
256
256
  self.watcher_raft_address = None
257
257
  if self.state.peer.is_app_leader:
258
- self.charm._patroni.cleanup_raft_cluster()
258
+ self.charm.raft_manager.cleanup_raft_cluster()
259
259
  # Update Patroni configuration without the watcher
260
260
  self.charm.update_config()
261
261
 
@@ -424,7 +424,7 @@ class WatcherEventsHandler(Object):
424
424
  # Collect PostgreSQL unit endpoints using fresh IPs from unit relation data.
425
425
  # _units_ips reads directly from unit relation data (always fresh), while
426
426
  # _peer_members_ips reads from app peer data (may be stale after network disruptions).
427
- pg_endpoints: list[str] = sorted(self.charm._units_ips)
427
+ pg_endpoints: list[str] = sorted(self.state.units_ips)
428
428
  if not pg_endpoints:
429
429
  logger.warning("No PostgreSQL endpoints available")
430
430
  return
@@ -7,10 +7,15 @@ import logging
7
7
  import os
8
8
  import signal
9
9
  import subprocess
10
+ from contextlib import suppress
11
+ from importlib import resources
10
12
  from pathlib import Path
11
13
  from sys import version_info
12
14
  from typing import TYPE_CHECKING
13
15
 
16
+ with suppress(ImportError):
17
+ from charmlibs.systemd import daemon_reload, service_enable
18
+ from jinja2 import Template
14
19
  from ops import ActiveStatus
15
20
 
16
21
  from single_kernel_postgresql.config.enums import Substrates
@@ -20,6 +25,7 @@ from single_kernel_postgresql.config.literals import (
20
25
  OBSERVER_VM_LOG_FILE_PATH,
21
26
  )
22
27
  from single_kernel_postgresql.managers.base import BaseManager
28
+ from single_kernel_postgresql.utils import render_file
23
29
 
24
30
  if TYPE_CHECKING:
25
31
  from single_kernel_postgresql.core.state import CharmState
@@ -129,6 +135,38 @@ class ObserverManager(BaseManager):
129
135
  except OSError:
130
136
  pass
131
137
 
138
+ def start_raft_observer(self) -> None:
139
+ """Render systemd units and start the observer."""
140
+ timer_service_file = "/etc/systemd/system/raft-observer.timer"
141
+ oneshot_service_file = "/etc/systemd/system/raft-observer.service"
142
+
143
+ template = Template(
144
+ resources
145
+ .files("single_kernel_postgresql.templates.vm")
146
+ .joinpath("raft-observer.service.j2")
147
+ .read_text()
148
+ )
149
+
150
+ rendered = template.render(
151
+ envvars=copy_environment(),
152
+ script="-m single_kernel_postgresql.scripts.raft_observer",
153
+ )
154
+ render_file(Substrates.VM, oneshot_service_file, rendered, 0o644, change_owner=False)
155
+
156
+ template = Template(
157
+ resources
158
+ .files("single_kernel_postgresql.templates.vm")
159
+ .joinpath("raft-observer.timer.j2")
160
+ .read_text()
161
+ )
162
+ rendered = template.render()
163
+ render_file(Substrates.VM, timer_service_file, rendered, 0o644, change_owner=False)
164
+
165
+ # Reload systemd to pick up the new service
166
+ daemon_reload()
167
+ service_enable(timer_service_file, "--now")
168
+ logger.info("Installed and enabled raft observer timer")
169
+
132
170
 
133
171
  def copy_environment() -> dict[str, str]:
134
172
  """Environment variables to pass to a script."""
@@ -206,8 +206,8 @@ class PatroniManager(BaseManager):
206
206
  endpoints = []
207
207
  if self.state.endpoint:
208
208
  endpoints.append(self.state.endpoint)
209
- for peer in self.state.endpoints:
210
- endpoints.append(peer)
209
+ for peer in self.state.endpoints:
210
+ endpoints.append(peer)
211
211
  # Request info from cluster endpoint (which returns all members of the cluster).
212
212
  if response := parallel_patroni_get_request(
213
213
  f"/{PATRONI_CLUSTER_STATUS_ENDPOINT}",
@@ -0,0 +1,284 @@
1
+ #!/usr/bin/env python3
2
+ # Copyright 2021 Canonical Ltd.
3
+ # See LICENSE file for licensing details.
4
+
5
+ """Raft manager for VM substrate."""
6
+
7
+ import logging
8
+ import shutil
9
+ from contextlib import suppress
10
+ from typing import TYPE_CHECKING
11
+
12
+ with suppress(ImportError):
13
+ import psutil
14
+ from pysyncobj.utility import TcpUtility, UtilityException
15
+ from ops import BlockedStatus
16
+ from tenacity import Retrying, wait_fixed
17
+
18
+ from single_kernel_postgresql.config.exceptions import (
19
+ AddRaftMemberFailedError,
20
+ RaftNotPromotedError,
21
+ RaftPostgresqlNotUpError,
22
+ RaftPostgresqlStillUpError,
23
+ RemoveRaftMemberFailedError,
24
+ )
25
+ from single_kernel_postgresql.config.literals import PEER_RELATION, RAFT_PARTNER_PREFIX, RAFT_PORT
26
+ from single_kernel_postgresql.managers.base import BaseManager
27
+
28
+ if TYPE_CHECKING:
29
+ from collections.abc import Callable
30
+
31
+ from ops import StatusBase
32
+
33
+ from single_kernel_postgresql.core.state import CharmState
34
+ from single_kernel_postgresql.events.watcher import WatcherEventsHandler
35
+ from single_kernel_postgresql.managers.patroni import PatroniManager
36
+ from single_kernel_postgresql.workload.base import BaseWorkload
37
+
38
+ logger = logging.getLogger(__name__)
39
+
40
+
41
+ class RaftManager(BaseManager):
42
+ """RAFT manager for VM substrate."""
43
+
44
+ def __init__(
45
+ self,
46
+ state: "CharmState",
47
+ workload: "BaseWorkload",
48
+ patroni_manager: "PatroniManager",
49
+ # TODO split off into a manager
50
+ watcher_handler: "WatcherEventsHandler",
51
+ update_config,
52
+ set_unit_status: "Callable[[StatusBase], None]",
53
+ peer_relation_changed,
54
+ remove_from_members_ips,
55
+ ) -> None:
56
+ super().__init__(state, workload, "raft_manager")
57
+ self.state = state
58
+ self.workload = workload
59
+ self.patroni_manager = patroni_manager
60
+ self.watcher_handler = watcher_handler
61
+
62
+ # Charm fucts
63
+ self.update_config = update_config
64
+ self.set_unit_status = set_unit_status
65
+ self.peer_relation_changed = peer_relation_changed
66
+ self.remove_from_members_ips = remove_from_members_ips
67
+
68
+ def has_raft_quorum(self) -> bool:
69
+ """Check if raft cluster has quorum."""
70
+ # Get the status of the raft cluster.
71
+ syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
72
+
73
+ raft_host = "127.0.0.1:2222"
74
+ try:
75
+ raft_status = syncobj_util.executeCommand(raft_host, ["status"])
76
+ except UtilityException:
77
+ logger.warning("Has raft quorum: Cannot connect to raft cluster")
78
+ return False
79
+ if not raft_status:
80
+ logger.warning("Has raft quorum: No status reported")
81
+ return False
82
+ return raft_status["has_quorum"]
83
+
84
+ def remove_raft_data(self) -> None:
85
+ """Stops Patroni and removes the raft journals."""
86
+ logger.info("Stopping patroni")
87
+ self.patroni_manager.stop_patroni()
88
+
89
+ logger.info("Wait for postgresql to stop")
90
+ for attempt in Retrying(wait=wait_fixed(5)):
91
+ with attempt:
92
+ for proc in psutil.process_iter(["name"]):
93
+ if proc.name() == "postgres":
94
+ raise RaftPostgresqlStillUpError()
95
+
96
+ logger.info("Removing raft data")
97
+ try:
98
+ path = self.workload.paths.patroni_conf / "raft"
99
+ if path.exists() and path.is_dir():
100
+ shutil.rmtree(str(path))
101
+ except OSError as e:
102
+ raise Exception(
103
+ f"Failed to remove previous cluster information with error: {e!s}"
104
+ ) from e
105
+ logger.info("Raft ready to reinitialise")
106
+
107
+ def reinitialise_raft_data(self) -> None:
108
+ """Reinitialise the raft journals and promoting the unit to leader. Should only be run on sync replicas."""
109
+ logger.info("Rerendering patroni config without peers")
110
+ # TODO factor out
111
+ self.update_config(no_peers=True)
112
+ logger.info("Starting patroni")
113
+ self.patroni_manager.start_patroni()
114
+
115
+ logger.info("Waiting for new raft cluster to initialise")
116
+ for attempt in Retrying(wait=wait_fixed(5)):
117
+ with attempt:
118
+ health_status = self.patroni_manager.get_patroni_health()
119
+ if (
120
+ health_status["role"] not in ["leader", "master"]
121
+ or health_status["state"] != "running"
122
+ ):
123
+ raise RaftNotPromotedError()
124
+
125
+ logger.info("Restarting patroni")
126
+ self.patroni_manager.restart_patroni()
127
+ for attempt in Retrying(wait=wait_fixed(5)):
128
+ with attempt:
129
+ found_postgres = False
130
+ for proc in psutil.process_iter(["name"]):
131
+ if proc.name() == "postgres":
132
+ found_postgres = True
133
+ break
134
+ if not found_postgres:
135
+ raise RaftPostgresqlNotUpError()
136
+ logger.info("Raft should be unstuck")
137
+
138
+ def cleanup_raft_cluster(self) -> bool:
139
+ """Cleanup RAFT members not belonging to the current cluster or not a related watcher."""
140
+ # Get Raft cluster status to find all members
141
+ try:
142
+ if not self.workload.is_patroni_running():
143
+ logger.warning("Raft cleanup: Patroni service not running.")
144
+ return True
145
+ syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
146
+ if raft_status := syncobj_util.executeCommand(f"127.0.0.1:{RAFT_PORT}", ["status"]):
147
+ # Find all partner nodes in the Raft cluster
148
+ # Keys look like: partner_node_status_server_10.131.50.142:2222
149
+ for key in raft_status:
150
+ if key.startswith(RAFT_PARTNER_PREFIX) and raft_status[key] != 2:
151
+ member_addr = key.replace(RAFT_PARTNER_PREFIX, "")
152
+ member_ip = member_addr.split(":")[0]
153
+
154
+ # Check if this is a stale watcher (not a PostgreSQL node and not current watcher)
155
+ if (
156
+ member_ip not in self.state.units_ips
157
+ and member_addr != self.watcher_handler.watcher_raft_address
158
+ ):
159
+ logger.info(f"Removing stale Raft member: {member_addr}")
160
+ self.remove_raft_member(member_addr)
161
+ # TODO Factor out
162
+ self.remove_from_members_ips(member_ip)
163
+ return True
164
+ return False
165
+ except Exception as e:
166
+ logger.debug(f"Error during Raft cleanup: {e}")
167
+ return False
168
+
169
+ def _set_stuck_raft_flag(self) -> None:
170
+ # TODO factor out
171
+ self.set_unit_status(BlockedStatus("Raft majority loss, run: promote-to-primary"))
172
+ logger.warning("Remove raft member: Stuck raft cluster detected")
173
+ self.state.peer.data.update({"raft_stuck": "True"})
174
+
175
+ # Leader doesn't always trigger when changing it's own peer data.
176
+ if self.state.peer.is_app_leader:
177
+ # TODO Factor out
178
+ self.peer_relation_changed.emit(
179
+ unit=self.state.peer.unit,
180
+ app=self.state.peer.unit.app,
181
+ relation=self.state.model.get_relation(PEER_RELATION),
182
+ )
183
+
184
+ def get_raft_status(self) -> dict | None:
185
+ """Get local raft status."""
186
+ # Get the status of the raft cluster.
187
+ syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
188
+
189
+ raft_host = f"127.0.0.1:{RAFT_PORT}"
190
+ with suppress(UtilityException):
191
+ return syncobj_util.executeCommand(raft_host, ["status"])
192
+
193
+ def remove_raft_member(
194
+ self, member_address: str, remote_address: str | None = None, set_raft_flags: bool = True
195
+ ) -> None:
196
+ """Remove a member from the raft cluster.
197
+
198
+ The raft cluster is a different cluster from the Patroni cluster.
199
+ It is responsible for defining which Patroni member can update
200
+ the primary member in the DCS.
201
+
202
+ Raises:
203
+ RaftMemberNotFoundError: if the member to be removed
204
+ is not part of the raft cluster.
205
+ """
206
+ if self.state.has_raft_keys():
207
+ logger.debug("Remove raft member: Raft already in recovery")
208
+ return
209
+
210
+ # Get the status of the raft cluster.
211
+ syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
212
+
213
+ raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}"
214
+ try:
215
+ raft_status = syncobj_util.executeCommand(raft_host, ["status"])
216
+ except UtilityException as e:
217
+ logger.warning("Remove raft member: Cannot connect to raft cluster")
218
+ raise RemoveRaftMemberFailedError() from e
219
+ if not raft_status:
220
+ logger.warning("Remove raft member: No raft status")
221
+ raise RemoveRaftMemberFailedError() from None
222
+
223
+ # Check whether the member is still part of the raft cluster.
224
+ if f"{RAFT_PARTNER_PREFIX}{member_address}" not in raft_status:
225
+ logger.debug("Remove raft member: Address already removed")
226
+ return
227
+
228
+ # If there's no quorum and the leader left raft cluster is stuck
229
+ if raft_status["has_quorum"] and not raft_status["leader"]:
230
+ logger.warning("Remove raft member: No raft leader")
231
+ raise RemoveRaftMemberFailedError() from None
232
+ if (
233
+ not raft_status["has_quorum"]
234
+ and (not raft_status["leader"] or raft_status["leader"].address == member_address)
235
+ and set_raft_flags
236
+ ):
237
+ self._set_stuck_raft_flag()
238
+ return
239
+
240
+ # Remove the member from the raft cluster.
241
+ try:
242
+ result = syncobj_util.executeCommand(raft_host, ["remove", member_address])
243
+ except UtilityException as e:
244
+ logger.debug("Remove raft member: Remove call failed")
245
+ raise RemoveRaftMemberFailedError() from e
246
+
247
+ if not result or not result.startswith("SUCCESS"):
248
+ logger.debug(f"Remove raft member: Remove call not successful with {result}")
249
+ raise RemoveRaftMemberFailedError() from None
250
+
251
+ def add_raft_member(self, member_address: str, remote_address: str | None = None) -> None:
252
+ """Add a member to the raft cluster."""
253
+ if self.state.has_raft_keys():
254
+ logger.debug("Add raft member: Raft already in recovery")
255
+ return
256
+
257
+ # Get the status of the raft cluster.
258
+ syncobj_util = TcpUtility(password=self.state.application.raft_password, timeout=3)
259
+
260
+ raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}"
261
+ try:
262
+ raft_status = syncobj_util.executeCommand(raft_host, ["status"])
263
+ except UtilityException as e:
264
+ logger.warning("Add raft member: Cannot connect to raft cluster")
265
+ raise AddRaftMemberFailedError() from e
266
+
267
+ if not raft_status:
268
+ logger.warning("Add raft member: No raft status")
269
+ raise AddRaftMemberFailedError() from None
270
+
271
+ # Check whether the member is still part of the raft cluster.
272
+ if f"{RAFT_PARTNER_PREFIX}{member_address}" in raft_status:
273
+ logger.debug("Add raft member: Address already added")
274
+ return
275
+
276
+ try:
277
+ result = syncobj_util.executeCommand(raft_host, ["add", member_address])
278
+ except UtilityException as e:
279
+ logger.debug("Add raft member: Remove call failed")
280
+ raise AddRaftMemberFailedError() from e
281
+
282
+ if not result or not result.startswith("SUCCESS"):
283
+ logger.debug(f"Add raft member: Remove call not successful with {result}")
284
+ raise AddRaftMemberFailedError() from None
@@ -11,12 +11,14 @@ from pysyncobj.utility import TcpUtility
11
11
  from yaml import safe_load
12
12
 
13
13
  from single_kernel_postgresql.config.literals import (
14
+ BASE_SNAP_DIR,
14
15
  PATRONI_CONF_PATH,
15
16
  RAFT_PARTNER_PREFIX,
16
17
  RAFT_PORT,
18
+ SNAP_COMMON,
17
19
  )
18
20
 
19
- PATRONI_CONF_FILE_PATH = f"{PATRONI_CONF_PATH}/patroni.yaml"
21
+ PATRONI_CONF_FILE_PATH = f"{BASE_SNAP_DIR}/{SNAP_COMMON}{PATRONI_CONF_PATH}/patroni.yaml"
20
22
 
21
23
 
22
24
  def dispatch(custom_event):
@@ -0,0 +1,13 @@
1
+ [Unit]
2
+ Description=Patroni Raft observer service
3
+ After=snap.charmed-postgresql.patroni.service
4
+ Wants=network.target
5
+
6
+ [Service]
7
+ Type=oneshot
8
+ {% for var, val in envvars.items() -%}
9
+ Environment="{{ var }}={{ val }}"
10
+ {% endfor -%}
11
+ ExecStart=python3 {{ script }}
12
+ StandardOutput=journal
13
+ StandardError=journal
@@ -0,0 +1,10 @@
1
+ [Unit]
2
+ Description=Patroni Raft observer timer
3
+
4
+ [Timer]
5
+ OnActiveSec=10s
6
+ OnUnitInactiveSec=10s
7
+ AccuracySec=1s
8
+
9
+ [Install]
10
+ WantedBy=timers.target