babelqueue 1.7.0__tar.gz → 1.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babelqueue-1.7.0 → babelqueue-1.9.0}/.github/workflows/ci.yml +3 -3
- {babelqueue-1.7.0 → babelqueue-1.9.0}/PKG-INFO +3 -1
- {babelqueue-1.7.0 → babelqueue-1.9.0}/pyproject.toml +2 -1
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/__init__.py +2 -1
- babelqueue-1.9.0/src/babelqueue/otel.py +176 -0
- babelqueue-1.9.0/src/babelqueue/redrive.py +165 -0
- babelqueue-1.9.0/tests/test_otel.py +131 -0
- babelqueue-1.9.0/tests/test_redrive.py +139 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/.github/FUNDING.yml +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/.gitignore +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/CHANGELOG.md +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/LICENSE +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/README.md +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/app.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/artemis_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/asb_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/idempotency.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/kafka_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/pulsar_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/schema.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/sqs_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/manifest.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_app.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_artemis_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_artemis_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_asb_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_asb_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_celery.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_codec.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_django.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_idempotency.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_kafka_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_kafka_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_pulsar_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_pulsar_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_redis_transport.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_schema.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_schema_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_sqs_conformance.py +0 -0
- {babelqueue-1.7.0 → babelqueue-1.9.0}/tests/test_sqs_transport.py +0 -0
|
@@ -27,7 +27,7 @@ jobs:
|
|
|
27
27
|
- name: Install (with Celery + Django adapters)
|
|
28
28
|
run: |
|
|
29
29
|
python -m pip install --upgrade pip
|
|
30
|
-
pip install -e ".[dev,celery,django]"
|
|
30
|
+
pip install -e ".[dev,celery,django,otel]" opentelemetry-sdk
|
|
31
31
|
|
|
32
32
|
- name: Run tests
|
|
33
33
|
run: pytest
|
|
@@ -44,7 +44,7 @@ jobs:
|
|
|
44
44
|
- name: Install (dev + all adapters for type context)
|
|
45
45
|
run: |
|
|
46
46
|
python -m pip install --upgrade pip
|
|
47
|
-
pip install -e ".[dev,celery,django,redis,amqp,sqs,azureservicebus,artemis]"
|
|
47
|
+
pip install -e ".[dev,celery,django,redis,amqp,sqs,azureservicebus,artemis,otel]" opentelemetry-sdk
|
|
48
48
|
- name: Ruff
|
|
49
49
|
run: ruff check src tests
|
|
50
50
|
- name: Mypy
|
|
@@ -89,7 +89,7 @@ jobs:
|
|
|
89
89
|
- name: Install (all adapters — full coverage with brokers)
|
|
90
90
|
run: |
|
|
91
91
|
python -m pip install --upgrade pip
|
|
92
|
-
pip install -e ".[redis,amqp,sqs,azureservicebus,artemis,celery,django,dev]"
|
|
92
|
+
pip install -e ".[redis,amqp,sqs,azureservicebus,artemis,celery,django,dev,otel]" opentelemetry-sdk
|
|
93
93
|
|
|
94
94
|
- name: Wait for ElasticMQ
|
|
95
95
|
run: |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.9.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -40,6 +40,8 @@ Provides-Extra: django
|
|
|
40
40
|
Requires-Dist: django>=4.2; extra == 'django'
|
|
41
41
|
Provides-Extra: kafka
|
|
42
42
|
Requires-Dist: confluent-kafka>=2.3; extra == 'kafka'
|
|
43
|
+
Provides-Extra: otel
|
|
44
|
+
Requires-Dist: opentelemetry-api>=1.20; extra == 'otel'
|
|
43
45
|
Provides-Extra: pulsar
|
|
44
46
|
Requires-Dist: pulsar-client>=3.4; extra == 'pulsar'
|
|
45
47
|
Provides-Extra: redis
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.9.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -36,6 +36,7 @@ azureservicebus = ["azure-servicebus>=7.11", "azure-identity>=1.15"]
|
|
|
36
36
|
pulsar = ["pulsar-client>=3.4"]
|
|
37
37
|
kafka = ["confluent-kafka>=2.3"]
|
|
38
38
|
artemis = ["python-qpid-proton>=0.39"]
|
|
39
|
+
otel = ["opentelemetry-api>=1.20"]
|
|
39
40
|
celery = ["celery>=5"]
|
|
40
41
|
django = ["django>=4.2"]
|
|
41
42
|
dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]
|
|
@@ -11,7 +11,7 @@ and dead-letter helpers. Framework adapters (Celery, Django, ...) build on this.
|
|
|
11
11
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
|
-
from . import dead_letter, idempotency
|
|
14
|
+
from . import dead_letter, idempotency, redrive
|
|
15
15
|
from .app import BabelQueue
|
|
16
16
|
from .codec import SCHEMA_VERSION, SOURCE_LANG, EnvelopeCodec
|
|
17
17
|
from .contracts import HasTraceId, PolyglotMessage
|
|
@@ -37,6 +37,7 @@ __all__ = [
|
|
|
37
37
|
"UnknownUrnError",
|
|
38
38
|
"dead_letter",
|
|
39
39
|
"idempotency",
|
|
40
|
+
"redrive",
|
|
40
41
|
"IdempotencyStore",
|
|
41
42
|
"InMemoryStore",
|
|
42
43
|
"__version__",
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
"""Optional OpenTelemetry tracing (ADR-0025): produce/consume spans correlated across hops.
|
|
2
|
+
|
|
3
|
+
The Python mirror of the Go ``babelqueue-go/otel`` module. It emits a CONSUMER span per
|
|
4
|
+
handled message and a PRODUCER span per publish, correlating them across every hop and SDK
|
|
5
|
+
through the envelope's ``trace_id`` — a UUID, which maps 1:1 to a 128-bit OTel trace id. The
|
|
6
|
+
wire envelope is untouched (GR-1) and the core never imports OpenTelemetry: this module is
|
|
7
|
+
only importable with the ``[otel]`` extra (``pip install babelqueue[otel]``), exactly like the
|
|
8
|
+
optional transport drivers.
|
|
9
|
+
|
|
10
|
+
from opentelemetry import trace
|
|
11
|
+
from babelqueue import BabelQueue, otel
|
|
12
|
+
|
|
13
|
+
tracer = trace.get_tracer("orders")
|
|
14
|
+
app = BabelQueue("redis://localhost:6379/0", queue="orders")
|
|
15
|
+
app.register("urn:babel:orders:created", otel.wrap_handler(tracer, on_order_created))
|
|
16
|
+
# producer side:
|
|
17
|
+
otel.publish(tracer, app, "urn:babel:orders:created", {"order_id": 1042})
|
|
18
|
+
|
|
19
|
+
Every hop that shares a ``trace_id`` shares one OTel trace. Exact cross-hop *span*
|
|
20
|
+
parent-child linkage (W3C ``traceparent`` as a transport header) is a documented follow-up.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import inspect
|
|
27
|
+
from typing import Any, Callable, Mapping, Optional
|
|
28
|
+
|
|
29
|
+
from opentelemetry.context import Context
|
|
30
|
+
from opentelemetry.trace import (
|
|
31
|
+
NonRecordingSpan,
|
|
32
|
+
SpanContext,
|
|
33
|
+
SpanKind,
|
|
34
|
+
TraceFlags,
|
|
35
|
+
Tracer,
|
|
36
|
+
set_span_in_context,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
Handler = Callable[..., None]
|
|
40
|
+
|
|
41
|
+
_SYSTEM = "babelqueue"
|
|
42
|
+
_MASK_128 = (1 << 128) - 1
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def trace_id_of(trace_id: str) -> int:
|
|
46
|
+
"""Map an envelope ``trace_id`` to a deterministic 128-bit OTel trace id.
|
|
47
|
+
|
|
48
|
+
A UUID maps to its 16 raw bytes; any other string is hashed (SHA-256, first 16 bytes).
|
|
49
|
+
The result is never zero (OTel's invalid trace id). The inverse of :func:`uuid_of` for
|
|
50
|
+
the UUID case.
|
|
51
|
+
"""
|
|
52
|
+
raw = _uuid_bytes(trace_id)
|
|
53
|
+
if raw is not None:
|
|
54
|
+
n = int.from_bytes(raw, "big")
|
|
55
|
+
if n != 0:
|
|
56
|
+
return n
|
|
57
|
+
digest = hashlib.sha256(trace_id.encode("utf-8")).digest()[:16]
|
|
58
|
+
return int.from_bytes(digest, "big") or 1
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def uuid_of(trace_id_int: int) -> str:
|
|
62
|
+
"""Format a 128-bit OTel trace id as a canonical UUID string.
|
|
63
|
+
|
|
64
|
+
The form a producer stamps into the message's ``trace_id`` so a consumer can recover the
|
|
65
|
+
same trace id via :func:`trace_id_of`.
|
|
66
|
+
"""
|
|
67
|
+
h = format(trace_id_int & _MASK_128, "032x")
|
|
68
|
+
return f"{h[0:8]}-{h[8:12]}-{h[12:16]}-{h[16:20]}-{h[20:32]}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _uuid_bytes(s: str) -> Optional[bytes]:
|
|
72
|
+
h = s.replace("-", "")
|
|
73
|
+
if len(h) != 32:
|
|
74
|
+
return None
|
|
75
|
+
try:
|
|
76
|
+
return bytes.fromhex(h)
|
|
77
|
+
except ValueError:
|
|
78
|
+
return None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _span_id_of(trace_id: str) -> int:
|
|
82
|
+
"""Derive a deterministic, non-zero 64-bit span id so the remote parent context is valid
|
|
83
|
+
(a span needs a valid parent to inherit a specific trace)."""
|
|
84
|
+
digest = hashlib.sha256(("babelqueue-span:" + trace_id).encode("utf-8")).digest()[:8]
|
|
85
|
+
return int.from_bytes(digest, "big") or 1
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parent_context(trace_id: str) -> Context:
|
|
89
|
+
sc = SpanContext(
|
|
90
|
+
trace_id=trace_id_of(trace_id),
|
|
91
|
+
span_id=_span_id_of(trace_id),
|
|
92
|
+
is_remote=True,
|
|
93
|
+
trace_flags=TraceFlags(TraceFlags.SAMPLED),
|
|
94
|
+
)
|
|
95
|
+
return set_span_in_context(NonRecordingSpan(sc))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _wants_envelope(fn: Handler) -> bool:
|
|
99
|
+
"""True if ``fn`` takes a 3rd positional arg (the full envelope) or ``*args``."""
|
|
100
|
+
try:
|
|
101
|
+
params = list(inspect.signature(fn).parameters.values())
|
|
102
|
+
except (TypeError, ValueError): # pragma: no cover - builtins / C callables
|
|
103
|
+
return False
|
|
104
|
+
positional = [
|
|
105
|
+
p for p in params if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)
|
|
106
|
+
]
|
|
107
|
+
return any(p.kind == p.VAR_POSITIONAL for p in params) or len(positional) >= 3
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def wrap_handler(tracer: Tracer, handler: Handler) -> Handler:
|
|
111
|
+
"""Wrap a consume handler to emit a CONSUMER span per message, in the OTel trace derived
|
|
112
|
+
from the envelope's ``trace_id``.
|
|
113
|
+
|
|
114
|
+
Register it like any handler: ``app.register(urn, wrap_handler(tracer, handler))``. The
|
|
115
|
+
wrapper's ``*args`` signature makes the runtime pass the full envelope (``data, meta,
|
|
116
|
+
envelope``), so it can read ``trace_id``/``job`` even when the inner handler only wants
|
|
117
|
+
``(data, meta)``. A raising handler records the exception on the span and re-raises, so the
|
|
118
|
+
runtime's retry / dead-letter path still applies.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def wrapped(*args: Any) -> None:
|
|
122
|
+
envelope = args[2] if len(args) > 2 and isinstance(args[2], Mapping) else {}
|
|
123
|
+
meta = args[1] if len(args) > 1 and isinstance(args[1], Mapping) else {}
|
|
124
|
+
trace_id = str(envelope.get("trace_id") or "")
|
|
125
|
+
urn = str(envelope.get("job") or envelope.get("urn") or "")
|
|
126
|
+
|
|
127
|
+
attributes: dict[str, Any] = {
|
|
128
|
+
"messaging.system": _SYSTEM,
|
|
129
|
+
"messaging.operation": "process",
|
|
130
|
+
"messaging.destination.name": str(meta.get("queue") or ""),
|
|
131
|
+
"messaging.message.id": str(meta.get("id") or ""),
|
|
132
|
+
"messaging.message.conversation_id": trace_id,
|
|
133
|
+
"messaging.babelqueue.attempts": int(envelope.get("attempts", 0) or 0),
|
|
134
|
+
}
|
|
135
|
+
context = _parent_context(trace_id) if trace_id else None
|
|
136
|
+
|
|
137
|
+
with tracer.start_as_current_span(
|
|
138
|
+
"process " + urn,
|
|
139
|
+
context=context,
|
|
140
|
+
kind=SpanKind.CONSUMER,
|
|
141
|
+
attributes=attributes,
|
|
142
|
+
):
|
|
143
|
+
if _wants_envelope(handler):
|
|
144
|
+
handler(*args)
|
|
145
|
+
else:
|
|
146
|
+
handler(args[0], args[1])
|
|
147
|
+
|
|
148
|
+
return wrapped
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def publish(
|
|
152
|
+
tracer: Tracer,
|
|
153
|
+
app: Any,
|
|
154
|
+
urn: str,
|
|
155
|
+
data: Mapping[str, Any],
|
|
156
|
+
*,
|
|
157
|
+
queue: Optional[str] = None,
|
|
158
|
+
) -> str:
|
|
159
|
+
"""Publish via a PRODUCER span ``publish <urn>``, carrying the active trace's id into the
|
|
160
|
+
message's ``trace_id`` so the downstream consumer recovers the same trace.
|
|
161
|
+
|
|
162
|
+
Behaves like ``app.publish`` (returns the message id); ``app`` is any object exposing
|
|
163
|
+
``publish(urn, data, *, queue=None, trace_id=None) -> str``.
|
|
164
|
+
"""
|
|
165
|
+
attributes = {
|
|
166
|
+
"messaging.system": _SYSTEM,
|
|
167
|
+
"messaging.operation": "publish",
|
|
168
|
+
"messaging.destination.name": urn,
|
|
169
|
+
}
|
|
170
|
+
with tracer.start_as_current_span(
|
|
171
|
+
"publish " + urn, kind=SpanKind.PRODUCER, attributes=attributes
|
|
172
|
+
) as span:
|
|
173
|
+
trace_id = uuid_of(span.get_span_context().trace_id)
|
|
174
|
+
message_id = app.publish(urn, data, queue=queue, trace_id=trace_id)
|
|
175
|
+
span.set_attribute("messaging.message.id", message_id)
|
|
176
|
+
return message_id
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Optional DLQ redrive tooling (ADR-0026): safe replay off the dead-letter queue.
|
|
2
|
+
|
|
3
|
+
The Python mirror of the Go ``Redrive``. It reads dead-lettered messages off a DLQ and
|
|
4
|
+
re-publishes each to its source queue (its ``dead_letter.original_queue``) or a chosen
|
|
5
|
+
``to_queue``, **reset for reprocessing**: the ``dead_letter`` block is removed and ``attempts``
|
|
6
|
+
reset to 0, while ``job``, ``trace_id``, ``data`` and ``meta`` are preserved verbatim. It is
|
|
7
|
+
the operator-side counterpart to the runtime's dead-letter routing — the contract leaves
|
|
8
|
+
redrive to tooling, and this is that tool.
|
|
9
|
+
|
|
10
|
+
from babelqueue import BabelQueue
|
|
11
|
+
from babelqueue.redrive import redrive
|
|
12
|
+
|
|
13
|
+
app = BabelQueue("redis://localhost:6379/0")
|
|
14
|
+
result = redrive(app.transport, "orders.dlq") # back to each source
|
|
15
|
+
result = redrive(app.transport, "orders.dlq", to_queue="sandbox") # safe sandbox replay
|
|
16
|
+
plan = redrive(app.transport, "orders.dlq", dry_run=True) # inspect, change nothing
|
|
17
|
+
|
|
18
|
+
Messages are drained from the DLQ first and then processed, so restored messages (skipped,
|
|
19
|
+
dry-run, or undecodable) are never re-encountered in the same run; a DLQ message is
|
|
20
|
+
acknowledged only after a successful re-publish, and an undecodable body is restored, not
|
|
21
|
+
dropped.
|
|
22
|
+
|
|
23
|
+
Replay safety today is sandbox routing (``to_queue``) + ``dry_run``. The **Replay-Bypass**
|
|
24
|
+
guard — a ``bq-replay-bypass`` transport header surfaced to handlers so a replay can skip
|
|
25
|
+
external side-effects (don't re-charge, don't re-email) — is a documented phase two: like the
|
|
26
|
+
OpenTelemetry ``traceparent`` follow-up, it carries out-of-band metadata as a transport header
|
|
27
|
+
and so touches the runtime + every transport binding. Until then, sandbox routing is the
|
|
28
|
+
safe-replay answer.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
|
|
35
|
+
|
|
36
|
+
from .codec import EnvelopeCodec
|
|
37
|
+
from .transport import ReceivedMessage, Transport
|
|
38
|
+
|
|
39
|
+
Envelope = Mapping[str, Any]
|
|
40
|
+
Select = Callable[[Envelope], bool]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class RedriveItem:
|
|
45
|
+
"""What happened to one message during a redrive run."""
|
|
46
|
+
|
|
47
|
+
message_id: str
|
|
48
|
+
trace_id: str
|
|
49
|
+
urn: str
|
|
50
|
+
reason: str
|
|
51
|
+
from_queue: str
|
|
52
|
+
to: str # target queue (the plan, even on a dry run; "" when skipped/undecodable)
|
|
53
|
+
redriven: bool # True only when actually re-published to ``to``
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass
|
|
57
|
+
class RedriveResult:
|
|
58
|
+
"""Summary of a redrive run."""
|
|
59
|
+
|
|
60
|
+
redriven: int = 0
|
|
61
|
+
skipped: int = 0
|
|
62
|
+
items: List[RedriveItem] = field(default_factory=list)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def redrive(
|
|
66
|
+
transport: Transport,
|
|
67
|
+
dlq: str,
|
|
68
|
+
*,
|
|
69
|
+
to_queue: Optional[str] = None,
|
|
70
|
+
max: int = 0,
|
|
71
|
+
dry_run: bool = False,
|
|
72
|
+
select: Optional[Select] = None,
|
|
73
|
+
timeout: float = 1.0,
|
|
74
|
+
) -> RedriveResult:
|
|
75
|
+
"""Move dead-lettered messages off ``dlq`` and replay them; see the module docstring."""
|
|
76
|
+
# Drain up to ``max`` messages (or all available) before processing any of them.
|
|
77
|
+
batch: List[Tuple[ReceivedMessage, Optional[Dict[str, Any]]]] = []
|
|
78
|
+
while max == 0 or len(batch) < max:
|
|
79
|
+
message = transport.pop(dlq, timeout)
|
|
80
|
+
if message is None:
|
|
81
|
+
break
|
|
82
|
+
batch.append((message, _decoded(message.body)))
|
|
83
|
+
|
|
84
|
+
result = RedriveResult()
|
|
85
|
+
for message, envelope in batch:
|
|
86
|
+
if envelope is None:
|
|
87
|
+
transport.publish(dlq, message.body) # restore the poison body; never drop it
|
|
88
|
+
transport.ack(message)
|
|
89
|
+
result.skipped += 1
|
|
90
|
+
result.items.append(RedriveItem("", "", "", "", dlq, "", False))
|
|
91
|
+
continue
|
|
92
|
+
|
|
93
|
+
meta_raw = envelope.get("meta")
|
|
94
|
+
meta: Mapping[str, Any] = meta_raw if isinstance(meta_raw, Mapping) else {}
|
|
95
|
+
dl_raw = envelope.get("dead_letter")
|
|
96
|
+
dead_letter: Mapping[str, Any] = dl_raw if isinstance(dl_raw, Mapping) else {}
|
|
97
|
+
item = RedriveItem(
|
|
98
|
+
message_id=str(meta.get("id", "")),
|
|
99
|
+
trace_id=str(envelope.get("trace_id", "")),
|
|
100
|
+
urn=EnvelopeCodec.urn(envelope),
|
|
101
|
+
reason=str(dead_letter.get("reason", "")),
|
|
102
|
+
from_queue=dlq,
|
|
103
|
+
to="",
|
|
104
|
+
redriven=False,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
if select is not None and not select(envelope):
|
|
108
|
+
transport.publish(dlq, message.body) # not selected: restore unchanged
|
|
109
|
+
transport.ack(message)
|
|
110
|
+
result.skipped += 1
|
|
111
|
+
result.items.append(item)
|
|
112
|
+
continue
|
|
113
|
+
|
|
114
|
+
target = to_queue or _source_queue_of(envelope)
|
|
115
|
+
item.to = target
|
|
116
|
+
|
|
117
|
+
if dry_run:
|
|
118
|
+
transport.publish(dlq, message.body) # report the plan; restore unchanged
|
|
119
|
+
transport.ack(message)
|
|
120
|
+
result.skipped += 1
|
|
121
|
+
result.items.append(item)
|
|
122
|
+
continue
|
|
123
|
+
|
|
124
|
+
reset = dict(envelope)
|
|
125
|
+
reset.pop("dead_letter", None)
|
|
126
|
+
reset["attempts"] = 0
|
|
127
|
+
try:
|
|
128
|
+
transport.publish(target, EnvelopeCodec.encode(reset))
|
|
129
|
+
except Exception:
|
|
130
|
+
transport.publish(dlq, message.body) # restore on a publish failure, then surface
|
|
131
|
+
transport.ack(message)
|
|
132
|
+
raise
|
|
133
|
+
transport.ack(message)
|
|
134
|
+
item.redriven = True
|
|
135
|
+
result.redriven += 1
|
|
136
|
+
result.items.append(item)
|
|
137
|
+
|
|
138
|
+
return result
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _decoded(body: str) -> Optional[Dict[str, Any]]:
|
|
142
|
+
"""Decode a DLQ body, or None when it is not a redrivable envelope.
|
|
143
|
+
|
|
144
|
+
``EnvelopeCodec.decode`` returns ``{}`` for malformed/non-object input; an object with no
|
|
145
|
+
string ``job`` is likewise not redrivable.
|
|
146
|
+
"""
|
|
147
|
+
envelope = EnvelopeCodec.decode(body)
|
|
148
|
+
if not envelope or not isinstance(envelope.get("job"), str):
|
|
149
|
+
return None
|
|
150
|
+
return envelope
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _source_queue_of(envelope: Envelope) -> str:
|
|
154
|
+
"""Default redrive target: ``dead_letter.original_queue``, falling back to ``meta.queue``."""
|
|
155
|
+
dead_letter = envelope.get("dead_letter")
|
|
156
|
+
if isinstance(dead_letter, Mapping):
|
|
157
|
+
original = dead_letter.get("original_queue")
|
|
158
|
+
if isinstance(original, str) and original:
|
|
159
|
+
return original
|
|
160
|
+
meta = envelope.get("meta")
|
|
161
|
+
if isinstance(meta, Mapping):
|
|
162
|
+
queue = meta.get("queue")
|
|
163
|
+
if isinstance(queue, str):
|
|
164
|
+
return queue
|
|
165
|
+
return ""
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Tests for the optional OpenTelemetry module (ADR-0025).
|
|
2
|
+
|
|
3
|
+
Skipped when OpenTelemetry is not installed (the ``[otel]`` extra); CI installs it so these
|
|
4
|
+
run and count toward coverage.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
|
|
9
|
+
pytest.importorskip("opentelemetry")
|
|
10
|
+
|
|
11
|
+
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
|
|
12
|
+
from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402
|
|
13
|
+
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
|
|
14
|
+
InMemorySpanExporter,
|
|
15
|
+
)
|
|
16
|
+
from opentelemetry.trace import SpanKind # noqa: E402
|
|
17
|
+
|
|
18
|
+
from babelqueue import otel # noqa: E402
|
|
19
|
+
|
|
20
|
+
_TRACE_ID = "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _recorder():
|
|
24
|
+
exporter = InMemorySpanExporter()
|
|
25
|
+
provider = TracerProvider()
|
|
26
|
+
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
|
27
|
+
return provider.get_tracer("test"), exporter
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _envelope(trace_id=_TRACE_ID, attempts=0):
|
|
31
|
+
return {
|
|
32
|
+
"job": "urn:babel:orders:created",
|
|
33
|
+
"trace_id": trace_id,
|
|
34
|
+
"data": {"order_id": 1},
|
|
35
|
+
"meta": {"id": "m1", "queue": "orders"},
|
|
36
|
+
"attempts": attempts,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class FakeApp:
|
|
41
|
+
"""An object exposing publish(urn, data, *, queue, trace_id) -> str, recording calls."""
|
|
42
|
+
|
|
43
|
+
def __init__(self):
|
|
44
|
+
self.calls = []
|
|
45
|
+
|
|
46
|
+
def publish(self, urn, data, *, queue=None, trace_id=None):
|
|
47
|
+
self.calls.append({"urn": urn, "data": data, "queue": queue, "trace_id": trace_id})
|
|
48
|
+
return "msg-123"
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_trace_id_round_trip():
|
|
52
|
+
tid = otel.trace_id_of(_TRACE_ID)
|
|
53
|
+
assert tid != 0
|
|
54
|
+
assert otel.uuid_of(tid) == _TRACE_ID
|
|
55
|
+
# a non-uuid trace_id maps deterministically to a valid, distinct trace id
|
|
56
|
+
assert otel.trace_id_of("not-a-uuid") == otel.trace_id_of("not-a-uuid")
|
|
57
|
+
assert otel.trace_id_of("not-a-uuid") != 0
|
|
58
|
+
assert otel.trace_id_of("not-a-uuid") != tid
|
|
59
|
+
# 32 chars but not hex → not a UUID, so it is hashed
|
|
60
|
+
assert otel.trace_id_of("z" * 32) != 0
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_wrap_handler_span_in_trace_with_attrs():
|
|
64
|
+
tracer, exporter = _recorder()
|
|
65
|
+
seen = {}
|
|
66
|
+
|
|
67
|
+
def handler(data, meta, envelope):
|
|
68
|
+
seen["called"] = True
|
|
69
|
+
|
|
70
|
+
env = _envelope()
|
|
71
|
+
otel.wrap_handler(tracer, handler)(env["data"], env["meta"], env)
|
|
72
|
+
|
|
73
|
+
assert seen.get("called")
|
|
74
|
+
spans = exporter.get_finished_spans()
|
|
75
|
+
assert len(spans) == 1
|
|
76
|
+
span = spans[0]
|
|
77
|
+
assert span.name == "process urn:babel:orders:created"
|
|
78
|
+
assert span.kind == SpanKind.CONSUMER
|
|
79
|
+
assert span.context.trace_id == otel.trace_id_of(_TRACE_ID)
|
|
80
|
+
assert span.attributes["messaging.message.conversation_id"] == _TRACE_ID
|
|
81
|
+
assert span.attributes["messaging.message.id"] == "m1"
|
|
82
|
+
assert span.attributes["messaging.destination.name"] == "orders"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_wrap_handler_two_arg_inner():
|
|
86
|
+
tracer, exporter = _recorder()
|
|
87
|
+
got = {}
|
|
88
|
+
|
|
89
|
+
def handler(data, meta): # only two positional args
|
|
90
|
+
got["data"] = data
|
|
91
|
+
|
|
92
|
+
env = _envelope()
|
|
93
|
+
# the runtime passes (data, meta, envelope); the wrapper forwards only what the inner wants
|
|
94
|
+
otel.wrap_handler(tracer, handler)(env["data"], env["meta"], env)
|
|
95
|
+
|
|
96
|
+
assert got["data"] == env["data"]
|
|
97
|
+
assert len(exporter.get_finished_spans()) == 1
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_wrap_handler_records_error():
|
|
101
|
+
tracer, exporter = _recorder()
|
|
102
|
+
|
|
103
|
+
def handler(data, meta, envelope):
|
|
104
|
+
raise ValueError("boom")
|
|
105
|
+
|
|
106
|
+
env = _envelope()
|
|
107
|
+
with pytest.raises(ValueError):
|
|
108
|
+
otel.wrap_handler(tracer, handler)(env["data"], env["meta"], env)
|
|
109
|
+
|
|
110
|
+
span = exporter.get_finished_spans()[0]
|
|
111
|
+
assert span.status.status_code.name == "ERROR"
|
|
112
|
+
assert len(span.events) >= 1 # the recorded exception
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_publish_stamps_trace_id_from_span():
|
|
116
|
+
tracer, exporter = _recorder()
|
|
117
|
+
app = FakeApp()
|
|
118
|
+
|
|
119
|
+
message_id = otel.publish(tracer, app, "urn:babel:orders:created", {"order_id": 7})
|
|
120
|
+
|
|
121
|
+
assert message_id == "msg-123"
|
|
122
|
+
spans = exporter.get_finished_spans()
|
|
123
|
+
assert len(spans) == 1
|
|
124
|
+
span = spans[0]
|
|
125
|
+
assert span.kind == SpanKind.PRODUCER
|
|
126
|
+
assert span.attributes["messaging.message.id"] == "msg-123"
|
|
127
|
+
|
|
128
|
+
stamped = app.calls[0]["trace_id"]
|
|
129
|
+
# the published trace_id encodes the producer span's trace, so a consumer recovers it
|
|
130
|
+
assert stamped == otel.uuid_of(span.context.trace_id)
|
|
131
|
+
assert otel.trace_id_of(stamped) == span.context.trace_id
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Tests for the optional DLQ redrive tooling (ADR-0026)."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from babelqueue import dead_letter
|
|
6
|
+
from babelqueue.codec import EnvelopeCodec
|
|
7
|
+
from babelqueue.redrive import redrive
|
|
8
|
+
from babelqueue.transport import InMemoryTransport
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _dead_letter(transport, dlq, urn, original_queue, data=None):
|
|
12
|
+
env = EnvelopeCodec.make(urn, data or {}, queue=original_queue)
|
|
13
|
+
annotated = dead_letter.annotate(env, "failed", original_queue, 3, error="boom")
|
|
14
|
+
transport.publish(dlq, EnvelopeCodec.encode(annotated))
|
|
15
|
+
return annotated
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _drain(transport, queue):
|
|
19
|
+
out = []
|
|
20
|
+
while True:
|
|
21
|
+
message = transport.pop(queue, 0)
|
|
22
|
+
if message is None:
|
|
23
|
+
break
|
|
24
|
+
out.append(EnvelopeCodec.decode(message.body))
|
|
25
|
+
transport.ack(message)
|
|
26
|
+
return out
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_redrive_to_source():
|
|
30
|
+
t = InMemoryTransport()
|
|
31
|
+
orig = _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders", {"order_id": 1})
|
|
32
|
+
|
|
33
|
+
result = redrive(t, "orders.dlq")
|
|
34
|
+
|
|
35
|
+
assert result.redriven == 1 and result.skipped == 0
|
|
36
|
+
got = _drain(t, "orders")
|
|
37
|
+
assert len(got) == 1
|
|
38
|
+
assert "dead_letter" not in got[0]
|
|
39
|
+
assert got[0]["attempts"] == 0
|
|
40
|
+
assert got[0]["trace_id"] == orig["trace_id"]
|
|
41
|
+
assert got[0]["data"] == {"order_id": 1}
|
|
42
|
+
assert EnvelopeCodec.urn(got[0]) == "urn:babel:orders:created"
|
|
43
|
+
assert _drain(t, "orders.dlq") == []
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_redrive_to_sandbox():
|
|
47
|
+
t = InMemoryTransport()
|
|
48
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
|
|
49
|
+
|
|
50
|
+
result = redrive(t, "orders.dlq", to_queue="sandbox")
|
|
51
|
+
|
|
52
|
+
assert result.redriven == 1
|
|
53
|
+
assert _drain(t, "orders") == []
|
|
54
|
+
assert len(_drain(t, "sandbox")) == 1
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def test_redrive_dry_run():
|
|
58
|
+
t = InMemoryTransport()
|
|
59
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
|
|
60
|
+
|
|
61
|
+
result = redrive(t, "orders.dlq", dry_run=True)
|
|
62
|
+
|
|
63
|
+
assert result.redriven == 0 and result.skipped == 1
|
|
64
|
+
assert result.items[0].to == "orders"
|
|
65
|
+
assert result.items[0].redriven is False
|
|
66
|
+
assert _drain(t, "orders") == []
|
|
67
|
+
dlq = _drain(t, "orders.dlq")
|
|
68
|
+
assert len(dlq) == 1 and "dead_letter" in dlq[0]
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def test_redrive_select():
|
|
72
|
+
t = InMemoryTransport()
|
|
73
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
74
|
+
_dead_letter(t, "dlq", "urn:babel:emails:welcome", "emails")
|
|
75
|
+
|
|
76
|
+
result = redrive(t, "dlq", select=lambda e: EnvelopeCodec.urn(e) == "urn:babel:orders:created")
|
|
77
|
+
|
|
78
|
+
assert result.redriven == 1 and result.skipped == 1
|
|
79
|
+
assert len(_drain(t, "orders")) == 1
|
|
80
|
+
assert _drain(t, "emails") == []
|
|
81
|
+
assert len(_drain(t, "dlq")) == 1 # the unselected one is restored
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_redrive_max():
|
|
85
|
+
t = InMemoryTransport()
|
|
86
|
+
for _ in range(3):
|
|
87
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
88
|
+
|
|
89
|
+
result = redrive(t, "dlq", max=2)
|
|
90
|
+
|
|
91
|
+
assert result.redriven == 2
|
|
92
|
+
assert len(_drain(t, "dlq")) == 1 # Max respected
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_redrive_no_dead_letter_falls_back_to_meta_queue():
|
|
96
|
+
t = InMemoryTransport()
|
|
97
|
+
# a plain (never dead-lettered) envelope on the DLQ — redrive falls back to meta.queue
|
|
98
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {}, queue="orders")
|
|
99
|
+
t.publish("dlq", EnvelopeCodec.encode(env))
|
|
100
|
+
|
|
101
|
+
result = redrive(t, "dlq")
|
|
102
|
+
|
|
103
|
+
assert result.redriven == 1
|
|
104
|
+
assert len(_drain(t, "orders")) == 1
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class _FailOnTarget(InMemoryTransport):
|
|
108
|
+
"""An in-memory transport that refuses to publish to one queue."""
|
|
109
|
+
|
|
110
|
+
def __init__(self, fail_queue):
|
|
111
|
+
super().__init__()
|
|
112
|
+
self._fail_queue = fail_queue
|
|
113
|
+
|
|
114
|
+
def publish(self, queue, body):
|
|
115
|
+
if queue == self._fail_queue:
|
|
116
|
+
raise RuntimeError("publish refused")
|
|
117
|
+
super().publish(queue, body)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_redrive_publish_failure_restores():
|
|
121
|
+
t = _FailOnTarget("orders")
|
|
122
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
123
|
+
|
|
124
|
+
with pytest.raises(RuntimeError):
|
|
125
|
+
redrive(t, "dlq")
|
|
126
|
+
|
|
127
|
+
assert len(_drain(t, "dlq")) == 1 # restored to the DLQ, not lost
|
|
128
|
+
assert _drain(t, "orders") == [] # nothing reached the source queue
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_redrive_undecodable_restored():
|
|
132
|
+
t = InMemoryTransport()
|
|
133
|
+
t.publish("dlq", "not-json{{{")
|
|
134
|
+
|
|
135
|
+
result = redrive(t, "dlq")
|
|
136
|
+
|
|
137
|
+
assert result.redriven == 0 and result.skipped == 1
|
|
138
|
+
message = t.pop("dlq", 0)
|
|
139
|
+
assert message is not None and message.body == "not-json{{{"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.7.0 → babelqueue-1.9.0}/src/babelqueue/django/management/commands/babelqueue_worker.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.7.0 → babelqueue-1.9.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|