rabbitkit 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rabbitkit/__init__.py +201 -0
- rabbitkit/_version.py +3 -0
- rabbitkit/aio/__init__.py +31 -0
- rabbitkit/async_/__init__.py +9 -0
- rabbitkit/async_/batch.py +213 -0
- rabbitkit/async_/broker.py +1123 -0
- rabbitkit/async_/connection.py +274 -0
- rabbitkit/async_/pool.py +363 -0
- rabbitkit/async_/transport.py +877 -0
- rabbitkit/asyncapi/__init__.py +5 -0
- rabbitkit/asyncapi/generator.py +219 -0
- rabbitkit/asyncapi/schema.py +98 -0
- rabbitkit/cli/__init__.py +77 -0
- rabbitkit/cli/_utils.py +38 -0
- rabbitkit/cli/commands/__init__.py +0 -0
- rabbitkit/cli/commands/dlq.py +190 -0
- rabbitkit/cli/commands/health.py +34 -0
- rabbitkit/cli/commands/migrate.py +570 -0
- rabbitkit/cli/commands/routes.py +88 -0
- rabbitkit/cli/commands/run.py +144 -0
- rabbitkit/cli/commands/shell.py +72 -0
- rabbitkit/cli/commands/topology.py +346 -0
- rabbitkit/concurrency.py +451 -0
- rabbitkit/core/__init__.py +5 -0
- rabbitkit/core/app.py +323 -0
- rabbitkit/core/config.py +849 -0
- rabbitkit/core/env_config.py +251 -0
- rabbitkit/core/errors.py +199 -0
- rabbitkit/core/logging.py +261 -0
- rabbitkit/core/message.py +235 -0
- rabbitkit/core/path.py +53 -0
- rabbitkit/core/pipeline.py +1289 -0
- rabbitkit/core/protocols.py +349 -0
- rabbitkit/core/registry.py +284 -0
- rabbitkit/core/route.py +329 -0
- rabbitkit/core/router.py +142 -0
- rabbitkit/core/topology.py +261 -0
- rabbitkit/core/topology_dispatch.py +74 -0
- rabbitkit/core/types.py +324 -0
- rabbitkit/dashboard/__init__.py +5 -0
- rabbitkit/dashboard/app.py +212 -0
- rabbitkit/di/__init__.py +19 -0
- rabbitkit/di/context.py +193 -0
- rabbitkit/di/depends.py +42 -0
- rabbitkit/di/resolver.py +503 -0
- rabbitkit/dlq.py +320 -0
- rabbitkit/experimental/__init__.py +50 -0
- rabbitkit/fastapi.py +91 -0
- rabbitkit/health.py +654 -0
- rabbitkit/highload/__init__.py +10 -0
- rabbitkit/highload/backpressure.py +514 -0
- rabbitkit/highload/batch.py +448 -0
- rabbitkit/locking.py +277 -0
- rabbitkit/management.py +470 -0
- rabbitkit/middleware/__init__.py +27 -0
- rabbitkit/middleware/base.py +125 -0
- rabbitkit/middleware/circuit_breaker.py +131 -0
- rabbitkit/middleware/compression.py +267 -0
- rabbitkit/middleware/deduplication.py +651 -0
- rabbitkit/middleware/error_classifier.py +43 -0
- rabbitkit/middleware/exception.py +105 -0
- rabbitkit/middleware/metrics.py +440 -0
- rabbitkit/middleware/otel.py +203 -0
- rabbitkit/middleware/rate_limit.py +247 -0
- rabbitkit/middleware/retry.py +540 -0
- rabbitkit/middleware/signing.py +682 -0
- rabbitkit/middleware/timeout.py +291 -0
- rabbitkit/py.typed +0 -0
- rabbitkit/queue_metrics.py +174 -0
- rabbitkit/results/__init__.py +6 -0
- rabbitkit/results/backend.py +102 -0
- rabbitkit/results/middleware.py +123 -0
- rabbitkit/rpc.py +632 -0
- rabbitkit/serialization/__init__.py +25 -0
- rabbitkit/serialization/base.py +35 -0
- rabbitkit/serialization/json.py +122 -0
- rabbitkit/serialization/msgspec.py +136 -0
- rabbitkit/serialization/pipeline.py +255 -0
- rabbitkit/streams.py +139 -0
- rabbitkit/sync/__init__.py +11 -0
- rabbitkit/sync/batch.py +595 -0
- rabbitkit/sync/broker.py +996 -0
- rabbitkit/sync/connection.py +209 -0
- rabbitkit/sync/pool.py +262 -0
- rabbitkit/sync/transport.py +1085 -0
- rabbitkit/testing/__init__.py +20 -0
- rabbitkit/testing/app.py +99 -0
- rabbitkit/testing/broker.py +540 -0
- rabbitkit/testing/fixtures.py +56 -0
- rabbitkit-0.9.0.dist-info/METADATA +575 -0
- rabbitkit-0.9.0.dist-info/RECORD +95 -0
- rabbitkit-0.9.0.dist-info/WHEEL +5 -0
- rabbitkit-0.9.0.dist-info/entry_points.txt +2 -0
- rabbitkit-0.9.0.dist-info/licenses/LICENSE +21 -0
- rabbitkit-0.9.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,570 @@
|
|
|
1
|
+
"""rabbitkit topology migrate — classic → quorum queue migration tool.
|
|
2
|
+
|
|
3
|
+
RabbitMQ cannot change ``x-queue-type`` in place: re-declaring an existing
|
|
4
|
+
classic queue as quorum fails with a 406 PRECONDITION_FAILED. This command
|
|
5
|
+
provides a supported migration path using dynamic shovels.
|
|
6
|
+
|
|
7
|
+
Modes
|
|
8
|
+
-----
|
|
9
|
+
* Default (``--plan``): read-only. Compares queues whose *declared* type is
|
|
10
|
+
quorum against the *live* broker; for each queue that is still classic,
|
|
11
|
+
prints an ordered runbook and writes a JSON snapshot (bindings + queue
|
|
12
|
+
arguments) as the rollback artifact. Never mutates.
|
|
13
|
+
* ``--execute --strategy drain-cutover``: performs the runbook via the
|
|
14
|
+
management API. Rails: refuses queues with consumers (unless ``--force``),
|
|
15
|
+
verifies message counts before every destructive step, and persists
|
|
16
|
+
progress to a state file after each completed step so a crashed run can
|
|
17
|
+
``--resume``.
|
|
18
|
+
* ``--execute --strategy bridge``: creates ``{queue}.q2`` quorum queues and
|
|
19
|
+
duplicates all bindings; prints instructions for moving consumers. Deletes
|
|
20
|
+
nothing.
|
|
21
|
+
* ``--execute --dry-run``: prints every management call it would make and
|
|
22
|
+
issues none of them (only read-only discovery calls are performed).
|
|
23
|
+
|
|
24
|
+
Requires the ``rabbitmq_shovel`` plugin for drain-cutover
|
|
25
|
+
(``rabbitmq-plugins enable rabbitmq_shovel``).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import json
|
|
31
|
+
import time
|
|
32
|
+
import urllib.parse
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import TYPE_CHECKING, Any, cast
|
|
35
|
+
|
|
36
|
+
import typer
|
|
37
|
+
|
|
38
|
+
from rabbitkit.cli._utils import load_broker
|
|
39
|
+
from rabbitkit.management import ManagementConfig, RabbitManagementClient
|
|
40
|
+
|
|
41
|
+
if TYPE_CHECKING:
|
|
42
|
+
from collections.abc import Callable
|
|
43
|
+
|
|
44
|
+
DEFAULT_MANAGEMENT_URL = "http://guest:guest@localhost:15672"
|
|
45
|
+
DEFAULT_AMQP_URL = "amqp://guest:guest@localhost/"
|
|
46
|
+
DEFAULT_STATE_FILE = ".rabbitkit-migrate.json"
|
|
47
|
+
DEFAULT_SNAPSHOT_FILE = "rabbitkit-migrate-snapshot.json"
|
|
48
|
+
|
|
49
|
+
_TMP_SUFFIX = ".migrate-tmp"
|
|
50
|
+
_POLL_INTERVAL = 1.0
|
|
51
|
+
_STRATEGIES = ("drain-cutover", "bridge")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _management_client(management_url: str) -> RabbitManagementClient:
|
|
55
|
+
"""Build a management client from a URL that may embed credentials.
|
|
56
|
+
|
|
57
|
+
Validates the scheme (http/https only — the URL feeds urllib directly)
|
|
58
|
+
and strips userinfo out of the base URL.
|
|
59
|
+
"""
|
|
60
|
+
parsed = urllib.parse.urlparse(management_url)
|
|
61
|
+
scheme = parsed.scheme.lower()
|
|
62
|
+
if scheme not in {"http", "https"}:
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Unsupported management URL scheme {scheme!r}; only 'http' and 'https' are allowed."
|
|
65
|
+
)
|
|
66
|
+
host = parsed.hostname or "localhost"
|
|
67
|
+
netloc = host if parsed.port is None else f"{host}:{parsed.port}"
|
|
68
|
+
base = f"{scheme}://{netloc}{parsed.path.rstrip('/')}"
|
|
69
|
+
return RabbitManagementClient(
|
|
70
|
+
ManagementConfig(
|
|
71
|
+
url=base,
|
|
72
|
+
username=parsed.username or "guest",
|
|
73
|
+
password=parsed.password or "guest",
|
|
74
|
+
)
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _declared_quorum_queues(broker: Any) -> set[str]:
|
|
79
|
+
"""Names of queues whose declared queue_type is quorum."""
|
|
80
|
+
names: set[str] = set()
|
|
81
|
+
for route in broker.routes:
|
|
82
|
+
q = route.queue
|
|
83
|
+
queue_type = getattr(q, "queue_type", None)
|
|
84
|
+
if getattr(queue_type, "value", queue_type) == "quorum":
|
|
85
|
+
names.add(q.name)
|
|
86
|
+
return names
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _live_classic_queues(client: RabbitManagementClient, vhost: str) -> dict[str, dict[str, Any]]:
|
|
90
|
+
"""Live queues whose actual x-queue-type is (implicitly or explicitly) classic."""
|
|
91
|
+
live: dict[str, dict[str, Any]] = {}
|
|
92
|
+
for queue_info in client.list_queues(vhost):
|
|
93
|
+
info = cast("dict[str, Any]", queue_info)
|
|
94
|
+
arguments = info.get("arguments") or {}
|
|
95
|
+
live_type = info.get("type") or arguments.get("x-queue-type") or "classic"
|
|
96
|
+
if live_type == "classic":
|
|
97
|
+
live[info["name"]] = info
|
|
98
|
+
return live
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _shovel_value(amqp_url: str, src: str, dest: str) -> dict[str, Any]:
|
|
102
|
+
"""Dynamic-shovel parameter body moving all current messages from src to dest."""
|
|
103
|
+
return {
|
|
104
|
+
"value": {
|
|
105
|
+
"src-protocol": "amqp091",
|
|
106
|
+
"src-uri": amqp_url,
|
|
107
|
+
"src-queue": src,
|
|
108
|
+
"dest-protocol": "amqp091",
|
|
109
|
+
"dest-uri": amqp_url,
|
|
110
|
+
"dest-queue": dest,
|
|
111
|
+
"src-delete-after": "queue-length",
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _require_shovel_plugin(client: RabbitManagementClient) -> None:
|
|
117
|
+
"""Fail fast when GET /api/shovels errors (plugin not enabled)."""
|
|
118
|
+
try:
|
|
119
|
+
client.list_shovel_statuses()
|
|
120
|
+
except Exception as exc:
|
|
121
|
+
typer.echo(
|
|
122
|
+
f"ERROR: the RabbitMQ shovel plugin is not available (GET /api/shovels failed: {exc}). "
|
|
123
|
+
"Enable it with: rabbitmq-plugins enable rabbitmq_shovel",
|
|
124
|
+
err=True,
|
|
125
|
+
)
|
|
126
|
+
raise typer.Exit(1) from None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _load_state(state_file: str) -> dict[str, Any]:
|
|
130
|
+
path = Path(state_file)
|
|
131
|
+
if path.exists():
|
|
132
|
+
return cast("dict[str, Any]", json.loads(path.read_text()))
|
|
133
|
+
return {"queues": {}}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _save_state(state_file: str, state: dict[str, Any]) -> None:
|
|
137
|
+
Path(state_file).write_text(json.dumps(state, indent=2))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _call(dry_run: bool, description: str, fn: Callable[..., Any], *args: Any, **kwargs: Any) -> None:
|
|
141
|
+
"""Issue a mutating management call, or just print it under --dry-run."""
|
|
142
|
+
if dry_run:
|
|
143
|
+
typer.echo(f" [dry-run] {description}")
|
|
144
|
+
return
|
|
145
|
+
typer.echo(f" {description}")
|
|
146
|
+
fn(*args, **kwargs)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _wait_empty(
|
|
150
|
+
client: RabbitManagementClient, name: str, vhost: str, timeout: float, dry_run: bool
|
|
151
|
+
) -> None:
|
|
152
|
+
"""Poll queue message count until zero, bounded by ``timeout`` seconds."""
|
|
153
|
+
if dry_run:
|
|
154
|
+
typer.echo(f" [dry-run] poll GET /api/queues/.../{name} until messages == 0 (timeout {timeout}s)")
|
|
155
|
+
return
|
|
156
|
+
deadline = time.monotonic() + timeout
|
|
157
|
+
while True:
|
|
158
|
+
info = cast("dict[str, Any]", client.get_queue(name, vhost))
|
|
159
|
+
messages = int(info.get("messages") or 0)
|
|
160
|
+
if messages == 0:
|
|
161
|
+
return
|
|
162
|
+
if time.monotonic() >= deadline:
|
|
163
|
+
typer.echo(
|
|
164
|
+
f"ERROR: timed out after {timeout}s waiting for queue '{name}' to drain "
|
|
165
|
+
f"({messages} message(s) left). Re-run with --resume once it is empty.",
|
|
166
|
+
err=True,
|
|
167
|
+
)
|
|
168
|
+
raise typer.Exit(1)
|
|
169
|
+
time.sleep(_POLL_INTERVAL)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _verify_empty(client: RabbitManagementClient, name: str, vhost: str, dry_run: bool) -> None:
|
|
173
|
+
"""Rail: re-verify the message count immediately before a destructive step."""
|
|
174
|
+
if dry_run:
|
|
175
|
+
return
|
|
176
|
+
info = cast("dict[str, Any]", client.get_queue(name, vhost))
|
|
177
|
+
messages = int(info.get("messages") or 0)
|
|
178
|
+
if messages:
|
|
179
|
+
typer.echo(
|
|
180
|
+
f"ERROR: refusing to delete '{name}': {messages} message(s) still queued.",
|
|
181
|
+
err=True,
|
|
182
|
+
)
|
|
183
|
+
raise typer.Exit(1)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _plan(
|
|
187
|
+
client: RabbitManagementClient,
|
|
188
|
+
candidates: list[str],
|
|
189
|
+
live: dict[str, dict[str, Any]],
|
|
190
|
+
vhost: str,
|
|
191
|
+
snapshot_file: str,
|
|
192
|
+
) -> None:
|
|
193
|
+
"""Print the ordered runbook and write the rollback snapshot. Never mutates."""
|
|
194
|
+
typer.echo(f"Migration plan — {len(candidates)} queue(s) need classic -> quorum migration.")
|
|
195
|
+
typer.echo("")
|
|
196
|
+
snapshot: dict[str, Any] = {"vhost": vhost, "queues": {}}
|
|
197
|
+
for name in candidates:
|
|
198
|
+
info = live[name]
|
|
199
|
+
bindings = client.get_queue_bindings(name, vhost)
|
|
200
|
+
snapshot["queues"][name] = {
|
|
201
|
+
"queue": {
|
|
202
|
+
"durable": info.get("durable", True),
|
|
203
|
+
"arguments": dict(info.get("arguments") or {}),
|
|
204
|
+
},
|
|
205
|
+
"bindings": bindings,
|
|
206
|
+
}
|
|
207
|
+
tmp = name + _TMP_SUFFIX
|
|
208
|
+
consumers = int(info.get("consumers") or 0)
|
|
209
|
+
n_bindings = len([b for b in bindings if b.get("source")])
|
|
210
|
+
steps = [
|
|
211
|
+
f"Verify zero consumers on '{name}' (currently {consumers})",
|
|
212
|
+
f"Snapshot bindings and queue arguments (rollback artifact: {snapshot_file})",
|
|
213
|
+
f"Create temporary quorum queue '{tmp}'",
|
|
214
|
+
f"Create shovel 'rabbitkit-migrate-{name}-out': '{name}' -> '{tmp}' (src-delete-after=queue-length)",
|
|
215
|
+
f"Wait until '{name}' is empty",
|
|
216
|
+
f"Delete classic queue '{name}'",
|
|
217
|
+
f"Redeclare '{name}' with x-queue-type=quorum (original arguments preserved)",
|
|
218
|
+
f"Recreate {n_bindings} binding(s) on '{name}'",
|
|
219
|
+
f"Create shovel 'rabbitkit-migrate-{name}-back': '{tmp}' -> '{name}'",
|
|
220
|
+
f"Wait until '{tmp}' is empty",
|
|
221
|
+
f"Delete temporary queue '{tmp}'",
|
|
222
|
+
]
|
|
223
|
+
typer.echo(f"Queue '{name}' (vhost {vhost!r}):")
|
|
224
|
+
for i, step in enumerate(steps, 1):
|
|
225
|
+
typer.echo(f" {i:2d}. {step}")
|
|
226
|
+
typer.echo("")
|
|
227
|
+
Path(snapshot_file).write_text(json.dumps(snapshot, indent=2))
|
|
228
|
+
typer.echo(f"Snapshot written to {snapshot_file} (rollback artifact).")
|
|
229
|
+
typer.echo("Re-run with --execute --strategy drain-cutover to perform the migration.")
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _bridge(
|
|
233
|
+
client: RabbitManagementClient, candidates: list[str], vhost: str, dry_run: bool
|
|
234
|
+
) -> None:
|
|
235
|
+
"""Create '{q}.q2' quorum queues with duplicated bindings. Deletes nothing."""
|
|
236
|
+
vhost_encoded = urllib.parse.quote(vhost, safe="")
|
|
237
|
+
for name in candidates:
|
|
238
|
+
bridge_queue = f"{name}.q2"
|
|
239
|
+
bindings = client.get_queue_bindings(name, vhost)
|
|
240
|
+
suffix = " [dry-run]" if dry_run else ""
|
|
241
|
+
typer.echo(f"Bridging '{name}' -> '{bridge_queue}' (quorum){suffix}:")
|
|
242
|
+
_call(
|
|
243
|
+
dry_run,
|
|
244
|
+
f"PUT /api/queues/{vhost_encoded}/{bridge_queue} (x-queue-type=quorum)",
|
|
245
|
+
client.declare_queue,
|
|
246
|
+
bridge_queue,
|
|
247
|
+
vhost=vhost,
|
|
248
|
+
durable=True,
|
|
249
|
+
arguments={"x-queue-type": "quorum"},
|
|
250
|
+
)
|
|
251
|
+
duplicated = 0
|
|
252
|
+
for binding in bindings:
|
|
253
|
+
source = binding.get("source") or ""
|
|
254
|
+
if not source: # default-exchange binding is implicit
|
|
255
|
+
continue
|
|
256
|
+
routing_key = binding.get("routing_key", "")
|
|
257
|
+
_call(
|
|
258
|
+
dry_run,
|
|
259
|
+
f"POST /api/bindings/{vhost_encoded}/e/{source}/q/{bridge_queue} (routing_key={routing_key!r})",
|
|
260
|
+
client.bind_queue,
|
|
261
|
+
bridge_queue,
|
|
262
|
+
source,
|
|
263
|
+
routing_key,
|
|
264
|
+
vhost,
|
|
265
|
+
binding.get("arguments") or None,
|
|
266
|
+
)
|
|
267
|
+
duplicated += 1
|
|
268
|
+
typer.echo(f" Duplicated {duplicated} binding(s) onto '{bridge_queue}'.")
|
|
269
|
+
typer.echo("")
|
|
270
|
+
typer.echo("Bridge queues created. Next steps (manual):")
|
|
271
|
+
typer.echo(" 1. Point consumers at the new '.q2' quorum queues and deploy them.")
|
|
272
|
+
typer.echo(" 2. Let the old classic queues drain (both queues receive new messages).")
|
|
273
|
+
typer.echo(" 3. Once drained, delete the old classic queues and (optionally) rename consumers.")
|
|
274
|
+
typer.echo("Nothing was deleted by this command.")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _drain_cutover_queue(
|
|
278
|
+
client: RabbitManagementClient,
|
|
279
|
+
name: str,
|
|
280
|
+
vhost: str,
|
|
281
|
+
amqp_url: str,
|
|
282
|
+
state: dict[str, Any],
|
|
283
|
+
state_file: str,
|
|
284
|
+
force: bool,
|
|
285
|
+
timeout: float,
|
|
286
|
+
dry_run: bool,
|
|
287
|
+
) -> None:
|
|
288
|
+
"""Run the shovel-based drain-cutover for a single queue, checkpointing each step."""
|
|
289
|
+
tmp = name + _TMP_SUFFIX
|
|
290
|
+
vhost_encoded = urllib.parse.quote(vhost, safe="")
|
|
291
|
+
qstate = state["queues"].setdefault(name, {})
|
|
292
|
+
completed: list[str] = qstate.setdefault("completed", [])
|
|
293
|
+
qstate.setdefault("snapshot", None)
|
|
294
|
+
|
|
295
|
+
def done(step: str) -> bool:
|
|
296
|
+
return step in completed
|
|
297
|
+
|
|
298
|
+
def mark(step: str) -> None:
|
|
299
|
+
completed.append(step)
|
|
300
|
+
if not dry_run:
|
|
301
|
+
_save_state(state_file, state)
|
|
302
|
+
|
|
303
|
+
suffix = " [dry-run]" if dry_run else ""
|
|
304
|
+
typer.echo(f"Migrating '{name}' (drain-cutover){suffix}:")
|
|
305
|
+
|
|
306
|
+
# 1. Rail: refuse to move a queue that still has consumers.
|
|
307
|
+
if not done("check-consumers"):
|
|
308
|
+
info = cast("dict[str, Any]", client.get_queue(name, vhost))
|
|
309
|
+
consumers = int(info.get("consumers") or 0)
|
|
310
|
+
if consumers > 0 and not force:
|
|
311
|
+
typer.echo(
|
|
312
|
+
f"ERROR: queue '{name}' has {consumers} consumer(s). "
|
|
313
|
+
"Stop them first, or pass --force to migrate anyway.",
|
|
314
|
+
err=True,
|
|
315
|
+
)
|
|
316
|
+
raise typer.Exit(1)
|
|
317
|
+
mark("check-consumers")
|
|
318
|
+
|
|
319
|
+
# 2. Snapshot bindings + arguments (rollback artifact, persisted in the state file).
|
|
320
|
+
if not done("snapshot") or qstate["snapshot"] is None:
|
|
321
|
+
bindings = client.get_queue_bindings(name, vhost)
|
|
322
|
+
info = cast("dict[str, Any]", client.get_queue(name, vhost))
|
|
323
|
+
qstate["snapshot"] = {
|
|
324
|
+
"queue": {
|
|
325
|
+
"durable": info.get("durable", True),
|
|
326
|
+
"arguments": dict(info.get("arguments") or {}),
|
|
327
|
+
},
|
|
328
|
+
"bindings": bindings,
|
|
329
|
+
}
|
|
330
|
+
if not done("snapshot"):
|
|
331
|
+
mark("snapshot")
|
|
332
|
+
elif not dry_run: # pragma: no cover — re-snapshot only when state was hand-edited
|
|
333
|
+
_save_state(state_file, state)
|
|
334
|
+
snapshot = qstate["snapshot"]
|
|
335
|
+
|
|
336
|
+
quorum_args = {k: v for k, v in snapshot["queue"]["arguments"].items() if k != "x-queue-type"}
|
|
337
|
+
quorum_args["x-queue-type"] = "quorum"
|
|
338
|
+
|
|
339
|
+
# 3. Temporary quorum queue that will hold messages during the cutover.
|
|
340
|
+
if not done("create-tmp"):
|
|
341
|
+
_call(
|
|
342
|
+
dry_run,
|
|
343
|
+
f"PUT /api/queues/{vhost_encoded}/{tmp} (x-queue-type=quorum)",
|
|
344
|
+
client.declare_queue,
|
|
345
|
+
tmp,
|
|
346
|
+
vhost=vhost,
|
|
347
|
+
durable=True,
|
|
348
|
+
arguments={"x-queue-type": "quorum"},
|
|
349
|
+
)
|
|
350
|
+
mark("create-tmp")
|
|
351
|
+
|
|
352
|
+
# 4. Shovel old -> tmp (auto-deletes itself after moving the initial queue length).
|
|
353
|
+
shovel_out = f"rabbitkit-migrate-{name}-out"
|
|
354
|
+
if not done("shovel-to-tmp"):
|
|
355
|
+
_call(
|
|
356
|
+
dry_run,
|
|
357
|
+
f"PUT /api/parameters/shovel/{vhost_encoded}/{shovel_out} ('{name}' -> '{tmp}')",
|
|
358
|
+
client.put_parameter,
|
|
359
|
+
"shovel",
|
|
360
|
+
vhost,
|
|
361
|
+
shovel_out,
|
|
362
|
+
_shovel_value(amqp_url, name, tmp),
|
|
363
|
+
)
|
|
364
|
+
mark("shovel-to-tmp")
|
|
365
|
+
|
|
366
|
+
# 5. Wait for the source to drain.
|
|
367
|
+
if not done("wait-source-empty"):
|
|
368
|
+
_wait_empty(client, name, vhost, timeout, dry_run)
|
|
369
|
+
mark("wait-source-empty")
|
|
370
|
+
|
|
371
|
+
# 6. Rail: re-verify emptiness, then delete the classic queue.
|
|
372
|
+
if not done("delete-source"):
|
|
373
|
+
_verify_empty(client, name, vhost, dry_run)
|
|
374
|
+
_call(
|
|
375
|
+
dry_run,
|
|
376
|
+
f"DELETE /api/queues/{vhost_encoded}/{name}",
|
|
377
|
+
client.delete_queue,
|
|
378
|
+
name,
|
|
379
|
+
vhost,
|
|
380
|
+
)
|
|
381
|
+
mark("delete-source")
|
|
382
|
+
|
|
383
|
+
# 7. Redeclare under the same name as quorum, preserving original arguments.
|
|
384
|
+
if not done("redeclare-quorum"):
|
|
385
|
+
_call(
|
|
386
|
+
dry_run,
|
|
387
|
+
f"PUT /api/queues/{vhost_encoded}/{name} (x-queue-type=quorum, original arguments)",
|
|
388
|
+
client.declare_queue,
|
|
389
|
+
name,
|
|
390
|
+
vhost=vhost,
|
|
391
|
+
durable=True,
|
|
392
|
+
arguments=quorum_args,
|
|
393
|
+
)
|
|
394
|
+
mark("redeclare-quorum")
|
|
395
|
+
|
|
396
|
+
# 8. Recreate the snapshotted bindings.
|
|
397
|
+
if not done("recreate-bindings"):
|
|
398
|
+
for binding in snapshot["bindings"]:
|
|
399
|
+
source = binding.get("source") or ""
|
|
400
|
+
if not source: # default-exchange binding is implicit
|
|
401
|
+
continue
|
|
402
|
+
routing_key = binding.get("routing_key", "")
|
|
403
|
+
_call(
|
|
404
|
+
dry_run,
|
|
405
|
+
f"POST /api/bindings/{vhost_encoded}/e/{source}/q/{name} (routing_key={routing_key!r})",
|
|
406
|
+
client.bind_queue,
|
|
407
|
+
name,
|
|
408
|
+
source,
|
|
409
|
+
routing_key,
|
|
410
|
+
vhost,
|
|
411
|
+
binding.get("arguments") or None,
|
|
412
|
+
)
|
|
413
|
+
mark("recreate-bindings")
|
|
414
|
+
|
|
415
|
+
# 9. Shovel tmp -> new quorum queue.
|
|
416
|
+
shovel_back = f"rabbitkit-migrate-{name}-back"
|
|
417
|
+
if not done("shovel-back"):
|
|
418
|
+
_call(
|
|
419
|
+
dry_run,
|
|
420
|
+
f"PUT /api/parameters/shovel/{vhost_encoded}/{shovel_back} ('{tmp}' -> '{name}')",
|
|
421
|
+
client.put_parameter,
|
|
422
|
+
"shovel",
|
|
423
|
+
vhost,
|
|
424
|
+
shovel_back,
|
|
425
|
+
_shovel_value(amqp_url, tmp, name),
|
|
426
|
+
)
|
|
427
|
+
mark("shovel-back")
|
|
428
|
+
|
|
429
|
+
# 10. Wait for tmp to drain.
|
|
430
|
+
if not done("wait-tmp-empty"):
|
|
431
|
+
_wait_empty(client, tmp, vhost, timeout, dry_run)
|
|
432
|
+
mark("wait-tmp-empty")
|
|
433
|
+
|
|
434
|
+
# 11. Rail: re-verify emptiness, then delete the temporary queue.
|
|
435
|
+
if not done("delete-tmp"):
|
|
436
|
+
_verify_empty(client, tmp, vhost, dry_run)
|
|
437
|
+
_call(
|
|
438
|
+
dry_run,
|
|
439
|
+
f"DELETE /api/queues/{vhost_encoded}/{tmp}",
|
|
440
|
+
client.delete_queue,
|
|
441
|
+
tmp,
|
|
442
|
+
vhost,
|
|
443
|
+
)
|
|
444
|
+
mark("delete-tmp")
|
|
445
|
+
|
|
446
|
+
typer.echo(f" Queue '{name}' migrated to quorum.")
|
|
447
|
+
|
|
448
|
+
|
|
449
|
+
def migrate_command(
|
|
450
|
+
app_path: str = typer.Argument(..., help="Broker path, e.g. 'myapp.main:broker'"),
|
|
451
|
+
management_url: str = typer.Option(
|
|
452
|
+
DEFAULT_MANAGEMENT_URL,
|
|
453
|
+
"--url",
|
|
454
|
+
"-u",
|
|
455
|
+
help="RabbitMQ management URL (may embed credentials)",
|
|
456
|
+
),
|
|
457
|
+
amqp_url: str = typer.Option(
|
|
458
|
+
DEFAULT_AMQP_URL,
|
|
459
|
+
"--amqp-url",
|
|
460
|
+
help="AMQP URI used as shovel src/dest URI",
|
|
461
|
+
),
|
|
462
|
+
vhost: str = typer.Option("/", "--vhost", "-v", help="Virtual host"),
|
|
463
|
+
strategy: str = typer.Option(
|
|
464
|
+
"drain-cutover",
|
|
465
|
+
"--strategy",
|
|
466
|
+
help="Migration strategy: drain-cutover or bridge",
|
|
467
|
+
),
|
|
468
|
+
plan: bool = typer.Option(False, "--plan", help="Print the runbook only (default mode)"),
|
|
469
|
+
execute: bool = typer.Option(False, "--execute", help="Perform the migration"),
|
|
470
|
+
dry_run: bool = typer.Option(
|
|
471
|
+
False, "--dry-run", help="With --execute: print every management call, issue none"
|
|
472
|
+
),
|
|
473
|
+
timeout: float = typer.Option(
|
|
474
|
+
300.0, "--timeout", help="Max seconds to wait for a queue to drain"
|
|
475
|
+
),
|
|
476
|
+
queue: str | None = typer.Option(None, "--queue", "-q", help="Limit migration to one queue"),
|
|
477
|
+
force: bool = typer.Option(
|
|
478
|
+
False, "--force", help="Proceed even if the queue has consumers"
|
|
479
|
+
),
|
|
480
|
+
resume: bool = typer.Option(
|
|
481
|
+
False, "--resume", help="Resume a crashed run from the state file"
|
|
482
|
+
),
|
|
483
|
+
state_file: str = typer.Option(
|
|
484
|
+
DEFAULT_STATE_FILE, "--state-file", help="Progress checkpoint file for --resume"
|
|
485
|
+
),
|
|
486
|
+
snapshot_file: str = typer.Option(
|
|
487
|
+
DEFAULT_SNAPSHOT_FILE, "--snapshot-file", help="Rollback snapshot file (plan mode)"
|
|
488
|
+
),
|
|
489
|
+
) -> None:
|
|
490
|
+
"""Migrate classic queues that are declared as quorum to actual quorum queues.
|
|
491
|
+
|
|
492
|
+
RabbitMQ cannot change x-queue-type in place — re-declaring 406s. Default
|
|
493
|
+
mode prints an ordered runbook and writes a rollback snapshot; --execute
|
|
494
|
+
performs it via the management API (requires the rabbitmq_shovel plugin
|
|
495
|
+
for the drain-cutover strategy).
|
|
496
|
+
|
|
497
|
+
Exit code 0 = success / nothing to do, 1 = refused or failed.
|
|
498
|
+
"""
|
|
499
|
+
if plan and execute:
|
|
500
|
+
typer.echo("ERROR: --plan and --execute are mutually exclusive.", err=True)
|
|
501
|
+
raise typer.Exit(1)
|
|
502
|
+
if dry_run and not execute:
|
|
503
|
+
typer.echo("ERROR: --dry-run only makes sense with --execute.", err=True)
|
|
504
|
+
raise typer.Exit(1)
|
|
505
|
+
if execute and strategy not in _STRATEGIES:
|
|
506
|
+
typer.echo(
|
|
507
|
+
f"ERROR: unknown strategy {strategy!r}; expected one of: {', '.join(_STRATEGIES)}.",
|
|
508
|
+
err=True,
|
|
509
|
+
)
|
|
510
|
+
raise typer.Exit(1)
|
|
511
|
+
|
|
512
|
+
try:
|
|
513
|
+
client = _management_client(management_url)
|
|
514
|
+
except ValueError as exc:
|
|
515
|
+
typer.echo(f"ERROR: {exc}", err=True)
|
|
516
|
+
raise typer.Exit(1) from None
|
|
517
|
+
|
|
518
|
+
broker = load_broker(app_path)
|
|
519
|
+
declared = _declared_quorum_queues(broker)
|
|
520
|
+
|
|
521
|
+
try:
|
|
522
|
+
live = _live_classic_queues(client, vhost)
|
|
523
|
+
except Exception as exc:
|
|
524
|
+
typer.echo(
|
|
525
|
+
f"ERROR: could not reach management API at {management_url}: {exc}", err=True
|
|
526
|
+
)
|
|
527
|
+
raise typer.Exit(1) from None
|
|
528
|
+
|
|
529
|
+
candidates = [name for name in sorted(declared) if name in live]
|
|
530
|
+
if queue is not None:
|
|
531
|
+
candidates = [name for name in candidates if name == queue]
|
|
532
|
+
if not candidates:
|
|
533
|
+
typer.echo("No queues need classic -> quorum migration.")
|
|
534
|
+
return
|
|
535
|
+
|
|
536
|
+
if not execute:
|
|
537
|
+
_plan(client, candidates, live, vhost, snapshot_file)
|
|
538
|
+
return
|
|
539
|
+
|
|
540
|
+
if strategy == "bridge":
|
|
541
|
+
_bridge(client, candidates, vhost, dry_run)
|
|
542
|
+
return
|
|
543
|
+
|
|
544
|
+
# drain-cutover
|
|
545
|
+
if dry_run:
|
|
546
|
+
typer.echo("[dry-run] GET /api/shovels (verify the rabbitmq_shovel plugin is enabled)")
|
|
547
|
+
else:
|
|
548
|
+
_require_shovel_plugin(client)
|
|
549
|
+
|
|
550
|
+
state = _load_state(state_file) if resume else {"queues": {}}
|
|
551
|
+
for name in candidates:
|
|
552
|
+
_drain_cutover_queue(
|
|
553
|
+
client,
|
|
554
|
+
name,
|
|
555
|
+
vhost=vhost,
|
|
556
|
+
amqp_url=amqp_url,
|
|
557
|
+
state=state,
|
|
558
|
+
state_file=state_file,
|
|
559
|
+
force=force,
|
|
560
|
+
timeout=timeout,
|
|
561
|
+
dry_run=dry_run,
|
|
562
|
+
)
|
|
563
|
+
|
|
564
|
+
if dry_run:
|
|
565
|
+
typer.echo(f"[dry-run] no changes were made; {len(candidates)} queue(s) would be migrated.")
|
|
566
|
+
else:
|
|
567
|
+
typer.echo(
|
|
568
|
+
f"Done — {len(candidates)} queue(s) migrated to quorum. "
|
|
569
|
+
f"State file: {state_file} (safe to delete)."
|
|
570
|
+
)
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""rabbitkit routes — inspect and describe registered consumer routes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
import typer
|
|
8
|
+
|
|
9
|
+
from rabbitkit.cli._utils import load_broker
|
|
10
|
+
|
|
11
|
+
routes_app = typer.Typer(help="Route inspection commands.")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@routes_app.command("list")
|
|
15
|
+
def routes_list(
|
|
16
|
+
app_path: str = typer.Argument(..., help="Broker path, e.g. 'myapp.main:broker'"),
|
|
17
|
+
output_format: str = typer.Option("table", "--format", "-f", help="Output format: table or json"),
|
|
18
|
+
) -> None:
|
|
19
|
+
"""List all registered consumer routes."""
|
|
20
|
+
broker = load_broker(app_path)
|
|
21
|
+
routes = broker.routes
|
|
22
|
+
|
|
23
|
+
rows = []
|
|
24
|
+
for r in routes:
|
|
25
|
+
retry = r.retry_config
|
|
26
|
+
rows.append({
|
|
27
|
+
"name": r.name,
|
|
28
|
+
"queue": r.queue.name,
|
|
29
|
+
"exchange": r.exchange.name if r.exchange else "",
|
|
30
|
+
"routing_key": r.queue.routing_key or r.queue.name,
|
|
31
|
+
"ack_policy": r.ack_policy.value,
|
|
32
|
+
"retry": f"{retry.max_retries}x" if retry and hasattr(retry, "max_retries") else "disabled",
|
|
33
|
+
"tags": ",".join(sorted(r.tags)) if r.tags else "",
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
if output_format == "json":
|
|
37
|
+
typer.echo(json.dumps(rows, indent=2))
|
|
38
|
+
return
|
|
39
|
+
|
|
40
|
+
if not rows:
|
|
41
|
+
typer.echo("No routes registered.")
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
headers = ["name", "queue", "exchange", "routing_key", "ack_policy", "retry"]
|
|
45
|
+
widths = {h: max(len(h), max((len(str(r.get(h, ""))) for r in rows), default=0)) for h in headers}
|
|
46
|
+
header_line = " ".join(h.ljust(widths[h]) for h in headers)
|
|
47
|
+
typer.echo(header_line)
|
|
48
|
+
typer.echo("-" * len(header_line))
|
|
49
|
+
for row in rows:
|
|
50
|
+
typer.echo(" ".join(str(row.get(h, "")).ljust(widths[h]) for h in headers))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@routes_app.command("describe")
|
|
54
|
+
def routes_describe(
|
|
55
|
+
app_path: str = typer.Argument(..., help="Broker path, e.g. 'myapp.main:broker'"),
|
|
56
|
+
route_name: str = typer.Argument(..., help="Route name to describe"),
|
|
57
|
+
) -> None:
|
|
58
|
+
"""Show full details of a single consumer route."""
|
|
59
|
+
broker = load_broker(app_path)
|
|
60
|
+
routes = broker.routes
|
|
61
|
+
|
|
62
|
+
match = next((r for r in routes if r.name == route_name), None)
|
|
63
|
+
if match is None:
|
|
64
|
+
typer.echo(f"Route '{route_name}' not found.", err=True)
|
|
65
|
+
raise typer.Exit(1)
|
|
66
|
+
|
|
67
|
+
retry = match.retry_config
|
|
68
|
+
info: dict[str, object] = {
|
|
69
|
+
"name": match.name,
|
|
70
|
+
"queue": {
|
|
71
|
+
"name": match.queue.name,
|
|
72
|
+
"routing_key": match.queue.routing_key or match.queue.name,
|
|
73
|
+
"durable": match.queue.durable,
|
|
74
|
+
"auto_delete": match.queue.auto_delete,
|
|
75
|
+
},
|
|
76
|
+
"exchange": {
|
|
77
|
+
"name": match.exchange.name,
|
|
78
|
+
"type": match.exchange.type.value if match.exchange else None,
|
|
79
|
+
} if match.exchange else None,
|
|
80
|
+
"ack_policy": match.ack_policy.value,
|
|
81
|
+
"retry": {
|
|
82
|
+
"max_retries": retry.max_retries,
|
|
83
|
+
"delays": list(retry.delays),
|
|
84
|
+
} if retry and hasattr(retry, "max_retries") else None,
|
|
85
|
+
"description": match.description,
|
|
86
|
+
"tags": sorted(match.tags) if match.tags else [],
|
|
87
|
+
}
|
|
88
|
+
typer.echo(json.dumps(info, indent=2))
|