rabbitmq-fxmacrodata 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rabbitmq_fxmacrodata-0.1.0/LICENSE +21 -0
- rabbitmq_fxmacrodata-0.1.0/MANIFEST.in +11 -0
- rabbitmq_fxmacrodata-0.1.0/PKG-INFO +129 -0
- rabbitmq_fxmacrodata-0.1.0/PUBLIC_FILES.json +11 -0
- rabbitmq_fxmacrodata-0.1.0/README.md +114 -0
- rabbitmq_fxmacrodata-0.1.0/examples/consumer_dedupe.py +29 -0
- rabbitmq_fxmacrodata-0.1.0/pyproject.toml +30 -0
- rabbitmq_fxmacrodata-0.1.0/setup.cfg +4 -0
- rabbitmq_fxmacrodata-0.1.0/src/fxmacrodata_rabbitmq/__init__.py +168 -0
- rabbitmq_fxmacrodata-0.1.0/src/fxmacrodata_rabbitmq/py.typed +0 -0
- rabbitmq_fxmacrodata-0.1.0/src/rabbitmq_fxmacrodata.egg-info/PKG-INFO +129 -0
- rabbitmq_fxmacrodata-0.1.0/src/rabbitmq_fxmacrodata.egg-info/SOURCES.txt +14 -0
- rabbitmq_fxmacrodata-0.1.0/src/rabbitmq_fxmacrodata.egg-info/dependency_links.txt +1 -0
- rabbitmq_fxmacrodata-0.1.0/src/rabbitmq_fxmacrodata.egg-info/requires.txt +5 -0
- rabbitmq_fxmacrodata-0.1.0/src/rabbitmq_fxmacrodata.egg-info/top_level.txt +1 -0
- rabbitmq_fxmacrodata-0.1.0/tests/test_publisher.py +169 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Robert Tidball
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
include LICENSE
|
|
2
|
+
include README.md
|
|
3
|
+
include pyproject.toml
|
|
4
|
+
include PUBLIC_FILES.json
|
|
5
|
+
include MANIFEST.in
|
|
6
|
+
include src/fxmacrodata_rabbitmq/__init__.py
|
|
7
|
+
include src/fxmacrodata_rabbitmq/py.typed
|
|
8
|
+
include tests/test_publisher.py
|
|
9
|
+
include examples/consumer_dedupe.py
|
|
10
|
+
global-exclude *.pyc *.pyo
|
|
11
|
+
global-exclude __pycache__/*
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rabbitmq-fxmacrodata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Publish public FXMacroData responses to RabbitMQ
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=package
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: fxmacrodata-platform-core==0.1.0
|
|
11
|
+
Requires-Dist: pika<2,>=1.4.4
|
|
12
|
+
Provides-Extra: test
|
|
13
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# FXMacroData for RabbitMQ
|
|
17
|
+
|
|
18
|
+
Publish FXMacroData public responses to a RabbitMQ exchange with the official
|
|
19
|
+
`pika` Python client. Route macro announcements to your own queues while
|
|
20
|
+
retaining the source payload and revision metadata.
|
|
21
|
+
|
|
22
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=readme)
|
|
23
|
+
|
|
24
|
+
## Install and connect
|
|
25
|
+
|
|
26
|
+
Requires Python 3.10 or later. Install this package and its adjacent public
|
|
27
|
+
client dependencies from the distribution supplied to you:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/rabbitmq-fxmacrodata
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Configure `RABBITMQ_URL` (an `amqps://` URL with the credentials your
|
|
34
|
+
administrator provides) in your runtime secret store. Then publish to a
|
|
35
|
+
customer-owned exchange:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import os
|
|
39
|
+
import pika
|
|
40
|
+
from fxmacrodata_platform_core import read
|
|
41
|
+
from fxmacrodata_rabbitmq import publish
|
|
42
|
+
|
|
43
|
+
connection = pika.BlockingConnection(pika.URLParameters(os.environ["RABBITMQ_URL"]))
|
|
44
|
+
try:
|
|
45
|
+
channel = connection.channel()
|
|
46
|
+
batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
|
|
47
|
+
result = publish(batch, channel, ("fxmacrodata", "usd.policy_rate"), confirm=True)
|
|
48
|
+
print(result.messages, "messages in", result.batches, "blocks")
|
|
49
|
+
finally:
|
|
50
|
+
connection.close()
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The exchange and any bound queues must already exist. The destination is an
|
|
54
|
+
`(exchange, routing_key)` tuple; names accept letters, digits, `_`, `.`, `:`
|
|
55
|
+
and `-` up to 255 characters. Use `("", "queue-name")` for the default
|
|
56
|
+
exchange and `("exchange-name", "")` for a fanout exchange. Wildcards, URLs
|
|
57
|
+
and other values are rejected before any publish.
|
|
58
|
+
|
|
59
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is
|
|
60
|
+
available, including REST and MCP operations. `publish_operation` reads one
|
|
61
|
+
operation and publishes it in a single call:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from fxmacrodata_rabbitmq import publish_operation
|
|
65
|
+
|
|
66
|
+
publish_operation("release_calendar", {"currency": "usd"}, channel, ("fxmacrodata", "usd.calendar"),
|
|
67
|
+
api_key_env="FXMD_API_KEY")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Message format
|
|
71
|
+
|
|
72
|
+
Each envelope row becomes one persistent message (`delivery_mode=2`). The
|
|
73
|
+
body is canonical UTF-8 JSON with the keys `operation`, `source_url`,
|
|
74
|
+
`record_id`, `currency`, `indicator`, `date`, `announcement_datetime`,
|
|
75
|
+
`value`, `value_text`, `record_json`, `response_json` and the publisher's
|
|
76
|
+
`content_hash` (SHA-256 of the ten source columns). Only `value` is numeric.
|
|
77
|
+
Dates and announcement times remain source strings. No publication time is
|
|
78
|
+
inferred, and no missing value is filled. `record_json` retains the original
|
|
79
|
+
record; `response_json` retains the complete response, including pagination
|
|
80
|
+
and metadata. An empty response produces one message with `record_json`
|
|
81
|
+
equal to `"null"`, so absence of data and the server's metadata remain
|
|
82
|
+
inspectable. It is not an observation.
|
|
83
|
+
|
|
84
|
+
`message_id` is `operation:record_id:content_hash`; the same source record
|
|
85
|
+
with the same content always produces the same id, and changed data or
|
|
86
|
+
metadata produces a different one. `content_type` is `application/json` and
|
|
87
|
+
the headers `operation`, `record_id`, `content_hash` and `source_url` are set
|
|
88
|
+
on every message. No timestamp property is derived from the data.
|
|
89
|
+
|
|
90
|
+
## Delivery guarantee
|
|
91
|
+
|
|
92
|
+
Delivery is at-least-once. Messages are published in blocks of `batch_size`
|
|
93
|
+
rows (1 to 10,000, 1,000 by default). With `confirm=True` the channel enters
|
|
94
|
+
publisher-confirm mode before the first publish, so a message the broker
|
|
95
|
+
rejects or cannot route fails its block; without confirms, a broker-side
|
|
96
|
+
failure surfaces only when the channel closes. A failed block raises
|
|
97
|
+
`PublishError` without the broker's own diagnostic text and is not retried;
|
|
98
|
+
earlier blocks may already be delivered. Consumers should deduplicate on
|
|
99
|
+
`message_id`, or on the `operation`, `record_id` and `content_hash` headers.
|
|
100
|
+
|
|
101
|
+
`max_message_bytes` (16 MiB by default) rejects an individual envelope larger
|
|
102
|
+
than the budget before anything is published; keep it within the broker's
|
|
103
|
+
`max_message_size`, and narrow the source date range to reduce the response
|
|
104
|
+
size. The source response is materialized before publishing; choose a source
|
|
105
|
+
date range suitable for memory.
|
|
106
|
+
|
|
107
|
+
`publish_operation` accepts `api_key_env`, `complete_history`, `max_pages`
|
|
108
|
+
and `client_factory` for the public source reader. Supply source credentials
|
|
109
|
+
through your runtime secret store, exposing only the environment variable
|
|
110
|
+
name; omit `api_key_env` for anonymous USD access. Destination authentication
|
|
111
|
+
belongs to the connection you open with the official client. The adapter
|
|
112
|
+
neither saves nor prints credentials and never places one in a message or
|
|
113
|
+
header.
|
|
114
|
+
|
|
115
|
+
The package licence covers this adapter code. FXMacroData data access and
|
|
116
|
+
onward redistribution remain subject to your subscription and applicable
|
|
117
|
+
licence. Publishing data to a broker does not grant permission to publish or
|
|
118
|
+
resell it.
|
|
119
|
+
|
|
120
|
+
## Development
|
|
121
|
+
|
|
122
|
+
Tests run offline against an in-memory fake channel:
|
|
123
|
+
|
|
124
|
+
```sh
|
|
125
|
+
python -m pytest tests -q
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Official SDK reference:
|
|
129
|
+
[pika BlockingChannel.basic_publish](https://pika.readthedocs.io/en/stable/modules/adapters/blocking.html?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=upstream-reference#pika.adapters.blocking_connection.BlockingChannel.basic_publish).
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# FXMacroData for RabbitMQ
|
|
2
|
+
|
|
3
|
+
Publish FXMacroData public responses to a RabbitMQ exchange with the official
|
|
4
|
+
`pika` Python client. Route macro announcements to your own queues while
|
|
5
|
+
retaining the source payload and revision metadata.
|
|
6
|
+
|
|
7
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=readme)
|
|
8
|
+
|
|
9
|
+
## Install and connect
|
|
10
|
+
|
|
11
|
+
Requires Python 3.10 or later. Install this package and its adjacent public
|
|
12
|
+
client dependencies from the distribution supplied to you:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/rabbitmq-fxmacrodata
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Configure `RABBITMQ_URL` (an `amqps://` URL with the credentials your
|
|
19
|
+
administrator provides) in your runtime secret store. Then publish to a
|
|
20
|
+
customer-owned exchange:
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
import os
|
|
24
|
+
import pika
|
|
25
|
+
from fxmacrodata_platform_core import read
|
|
26
|
+
from fxmacrodata_rabbitmq import publish
|
|
27
|
+
|
|
28
|
+
connection = pika.BlockingConnection(pika.URLParameters(os.environ["RABBITMQ_URL"]))
|
|
29
|
+
try:
|
|
30
|
+
channel = connection.channel()
|
|
31
|
+
batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
|
|
32
|
+
result = publish(batch, channel, ("fxmacrodata", "usd.policy_rate"), confirm=True)
|
|
33
|
+
print(result.messages, "messages in", result.batches, "blocks")
|
|
34
|
+
finally:
|
|
35
|
+
connection.close()
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The exchange and any bound queues must already exist. The destination is an
|
|
39
|
+
`(exchange, routing_key)` tuple; names accept letters, digits, `_`, `.`, `:`
|
|
40
|
+
and `-` up to 255 characters. Use `("", "queue-name")` for the default
|
|
41
|
+
exchange and `("exchange-name", "")` for a fanout exchange. Wildcards, URLs
|
|
42
|
+
and other values are rejected before any publish.
|
|
43
|
+
|
|
44
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is
|
|
45
|
+
available, including REST and MCP operations. `publish_operation` reads one
|
|
46
|
+
operation and publishes it in a single call:
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
from fxmacrodata_rabbitmq import publish_operation
|
|
50
|
+
|
|
51
|
+
publish_operation("release_calendar", {"currency": "usd"}, channel, ("fxmacrodata", "usd.calendar"),
|
|
52
|
+
api_key_env="FXMD_API_KEY")
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Message format
|
|
56
|
+
|
|
57
|
+
Each envelope row becomes one persistent message (`delivery_mode=2`). The
|
|
58
|
+
body is canonical UTF-8 JSON with the keys `operation`, `source_url`,
|
|
59
|
+
`record_id`, `currency`, `indicator`, `date`, `announcement_datetime`,
|
|
60
|
+
`value`, `value_text`, `record_json`, `response_json` and the publisher's
|
|
61
|
+
`content_hash` (SHA-256 of the ten source columns). Only `value` is numeric.
|
|
62
|
+
Dates and announcement times remain source strings. No publication time is
|
|
63
|
+
inferred, and no missing value is filled. `record_json` retains the original
|
|
64
|
+
record; `response_json` retains the complete response, including pagination
|
|
65
|
+
and metadata. An empty response produces one message with `record_json`
|
|
66
|
+
equal to `"null"`, so absence of data and the server's metadata remain
|
|
67
|
+
inspectable. It is not an observation.
|
|
68
|
+
|
|
69
|
+
`message_id` is `operation:record_id:content_hash`; the same source record
|
|
70
|
+
with the same content always produces the same id, and changed data or
|
|
71
|
+
metadata produces a different one. `content_type` is `application/json` and
|
|
72
|
+
the headers `operation`, `record_id`, `content_hash` and `source_url` are set
|
|
73
|
+
on every message. No timestamp property is derived from the data.
|
|
74
|
+
|
|
75
|
+
## Delivery guarantee
|
|
76
|
+
|
|
77
|
+
Delivery is at-least-once. Messages are published in blocks of `batch_size`
|
|
78
|
+
rows (1 to 10,000, 1,000 by default). With `confirm=True` the channel enters
|
|
79
|
+
publisher-confirm mode before the first publish, so a message the broker
|
|
80
|
+
rejects or cannot route fails its block; without confirms, a broker-side
|
|
81
|
+
failure surfaces only when the channel closes. A failed block raises
|
|
82
|
+
`PublishError` without the broker's own diagnostic text and is not retried;
|
|
83
|
+
earlier blocks may already be delivered. Consumers should deduplicate on
|
|
84
|
+
`message_id`, or on the `operation`, `record_id` and `content_hash` headers.
|
|
85
|
+
|
|
86
|
+
`max_message_bytes` (16 MiB by default) rejects an individual envelope larger
|
|
87
|
+
than the budget before anything is published; keep it within the broker's
|
|
88
|
+
`max_message_size`, and narrow the source date range to reduce the response
|
|
89
|
+
size. The source response is materialized before publishing; choose a source
|
|
90
|
+
date range suitable for memory.
|
|
91
|
+
|
|
92
|
+
`publish_operation` accepts `api_key_env`, `complete_history`, `max_pages`
|
|
93
|
+
and `client_factory` for the public source reader. Supply source credentials
|
|
94
|
+
through your runtime secret store, exposing only the environment variable
|
|
95
|
+
name; omit `api_key_env` for anonymous USD access. Destination authentication
|
|
96
|
+
belongs to the connection you open with the official client. The adapter
|
|
97
|
+
neither saves nor prints credentials and never places one in a message or
|
|
98
|
+
header.
|
|
99
|
+
|
|
100
|
+
The package licence covers this adapter code. FXMacroData data access and
|
|
101
|
+
onward redistribution remain subject to your subscription and applicable
|
|
102
|
+
licence. Publishing data to a broker does not grant permission to publish or
|
|
103
|
+
resell it.
|
|
104
|
+
|
|
105
|
+
## Development
|
|
106
|
+
|
|
107
|
+
Tests run offline against an in-memory fake channel:
|
|
108
|
+
|
|
109
|
+
```sh
|
|
110
|
+
python -m pytest tests -q
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Official SDK reference:
|
|
114
|
+
[pika BlockingChannel.basic_publish](https://pika.readthedocs.io/en/stable/modules/adapters/blocking.html?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=upstream-reference#pika.adapters.blocking_connection.BlockingChannel.basic_publish).
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Consume FXMacroData envelopes from a RabbitMQ queue, dropping redeliveries by message_id."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
import pika
|
|
6
|
+
|
|
7
|
+
seen: set[str] = set()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def on_message(channel, method, properties, body):
|
|
11
|
+
key = properties.message_id or ":".join(
|
|
12
|
+
str((properties.headers or {}).get(name, "")) for name in ("operation", "record_id", "content_hash")
|
|
13
|
+
)
|
|
14
|
+
if key not in seen:
|
|
15
|
+
seen.add(key)
|
|
16
|
+
row = json.loads(body.decode("utf-8"))
|
|
17
|
+
if row["record_json"] != "null":
|
|
18
|
+
print(row["operation"], row["currency"], row["indicator"], row["date"], row["value"])
|
|
19
|
+
channel.basic_ack(method.delivery_tag)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
connection = pika.BlockingConnection(pika.URLParameters(os.environ["RABBITMQ_URL"]))
|
|
23
|
+
try:
|
|
24
|
+
channel = connection.channel()
|
|
25
|
+
channel.basic_qos(prefetch_count=100)
|
|
26
|
+
channel.basic_consume("fxmacrodata-queue", on_message)
|
|
27
|
+
channel.start_consuming()
|
|
28
|
+
finally:
|
|
29
|
+
connection.close()
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "rabbitmq-fxmacrodata"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Publish public FXMacroData responses to RabbitMQ"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = ["fxmacrodata-platform-core==0.1.0", "pika>=1.4.4,<2"]
|
|
13
|
+
|
|
14
|
+
[project.urls]
|
|
15
|
+
Documentation = "https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=package"
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
test = ["pytest>=8,<9"]
|
|
19
|
+
|
|
20
|
+
[tool.setuptools]
|
|
21
|
+
packages = ["fxmacrodata_rabbitmq"]
|
|
22
|
+
package-dir = {"" = "src"}
|
|
23
|
+
include-package-data = false
|
|
24
|
+
|
|
25
|
+
[tool.setuptools.package-data]
|
|
26
|
+
fxmacrodata_rabbitmq = ["py.typed"]
|
|
27
|
+
|
|
28
|
+
[tool.pytest.ini_options]
|
|
29
|
+
testpaths = ["tests"]
|
|
30
|
+
addopts = "-q"
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Publish public FXMacroData responses to a customer-owned RabbitMQ exchange."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import pika
|
|
13
|
+
from fxmacrodata_platform_core import read
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
|
|
17
|
+
COLUMNS = (
|
|
18
|
+
"operation", "source_url", "record_id", "currency", "indicator", "date",
|
|
19
|
+
"announcement_datetime", "value", "value_text", "record_json", "response_json",
|
|
20
|
+
"content_hash",
|
|
21
|
+
)
|
|
22
|
+
HEADERS = ("operation", "record_id", "content_hash", "source_url")
|
|
23
|
+
MAX_MESSAGE_BYTES = 16 * 1024 * 1024
|
|
24
|
+
_NAME = re.compile(r"[A-Za-z0-9_.:-]{0,255}\Z")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class PublishError(RuntimeError):
|
|
28
|
+
"""A destination publish failed; inspect the destination before retrying."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class PublishResult:
|
|
33
|
+
destination: str
|
|
34
|
+
messages: int
|
|
35
|
+
batches: int
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _destination(destination: Any) -> tuple[str, str]:
|
|
39
|
+
if not isinstance(destination, tuple) or len(destination) != 2:
|
|
40
|
+
raise ValueError("destination must be an (exchange, routing_key) tuple")
|
|
41
|
+
exchange, routing_key = destination
|
|
42
|
+
if not isinstance(exchange, str) or not isinstance(routing_key, str):
|
|
43
|
+
raise ValueError("exchange and routing_key must be strings")
|
|
44
|
+
if not _NAME.fullmatch(exchange) or not _NAME.fullmatch(routing_key):
|
|
45
|
+
raise ValueError("exchange and routing_key accept letters, digits, '_', '.', ':' and '-' only")
|
|
46
|
+
if not exchange and not routing_key:
|
|
47
|
+
raise ValueError("the default exchange requires a routing_key naming the queue")
|
|
48
|
+
return exchange, routing_key
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _bounds(batch_size: int, max_message_bytes: int, confirm: bool) -> None:
|
|
52
|
+
if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
|
|
53
|
+
raise ValueError("batch_size must be an integer between 1 and 10000")
|
|
54
|
+
if isinstance(max_message_bytes, bool) or not isinstance(max_message_bytes, int) or not 1 <= max_message_bytes <= MAX_MESSAGE_BYTES:
|
|
55
|
+
raise ValueError(f"max_message_bytes must be an integer between 1 and {MAX_MESSAGE_BYTES}")
|
|
56
|
+
if not isinstance(confirm, bool):
|
|
57
|
+
raise ValueError("confirm must be a boolean")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _rows(batch: Any) -> list[dict[str, Any]]:
|
|
61
|
+
rows = []
|
|
62
|
+
for source in batch.rows():
|
|
63
|
+
row = {name: source.get(name) for name in COLUMNS[:-1]}
|
|
64
|
+
if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
|
|
65
|
+
raise ValueError("every row requires string operation and record_id")
|
|
66
|
+
for name, value in row.items():
|
|
67
|
+
if name == "value":
|
|
68
|
+
if value is not None and (
|
|
69
|
+
isinstance(value, bool) or not isinstance(value, (int, float)) or
|
|
70
|
+
not math.isfinite(value)
|
|
71
|
+
):
|
|
72
|
+
raise ValueError("value must be a finite number or null")
|
|
73
|
+
elif value is not None and not isinstance(value, str):
|
|
74
|
+
raise ValueError(f"{name} must be a string or null")
|
|
75
|
+
encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
76
|
+
row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
77
|
+
rows.append(row)
|
|
78
|
+
return rows
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def message_key(row: dict[str, Any]) -> str:
|
|
82
|
+
"""Return the idempotency key used as the AMQP message_id."""
|
|
83
|
+
return f"{row['operation']}:{row['record_id']}:{row['content_hash']}"
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def encode_message(row: dict[str, Any]) -> bytes:
|
|
87
|
+
"""Return the canonical UTF-8 JSON body for one envelope row."""
|
|
88
|
+
return json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
89
|
+
allow_nan=False).encode("utf-8")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _properties(row: dict[str, Any]) -> pika.BasicProperties:
|
|
93
|
+
return pika.BasicProperties(
|
|
94
|
+
delivery_mode=2, content_type="application/json", content_encoding="utf-8",
|
|
95
|
+
message_id=message_key(row),
|
|
96
|
+
headers={name: row[name] for name in HEADERS if row.get(name) is not None},
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _blocks(rows: list[dict[str, Any]], batch_size: int,
|
|
101
|
+
max_message_bytes: int) -> list[list[tuple[bytes, pika.BasicProperties]]]:
|
|
102
|
+
output: list[list[tuple[bytes, pika.BasicProperties]]] = []
|
|
103
|
+
block: list[tuple[bytes, pika.BasicProperties]] = []
|
|
104
|
+
for row in rows:
|
|
105
|
+
body = encode_message(row)
|
|
106
|
+
if len(body) > max_message_bytes:
|
|
107
|
+
raise ValueError("one envelope exceeds max_message_bytes; narrow the source date range")
|
|
108
|
+
if len(block) >= batch_size:
|
|
109
|
+
output.append(block)
|
|
110
|
+
block = []
|
|
111
|
+
block.append((body, _properties(row)))
|
|
112
|
+
if block:
|
|
113
|
+
output.append(block)
|
|
114
|
+
return output
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def publish(batch: Any, channel: Any, destination: tuple[str, str], *, batch_size: int = 1000,
|
|
118
|
+
max_message_bytes: int = MAX_MESSAGE_BYTES, confirm: bool = False) -> PublishResult:
|
|
119
|
+
"""Publish one persistent message per envelope row through BlockingChannel.basic_publish.
|
|
120
|
+
|
|
121
|
+
Messages carry the canonical JSON body, `message_id` equal to
|
|
122
|
+
`operation:record_id:content_hash`, `content_type` application/json and
|
|
123
|
+
the headers `operation`, `record_id`, `content_hash` and `source_url`.
|
|
124
|
+
With `confirm=True` the channel enters publisher-confirm mode before the
|
|
125
|
+
first publish, so an unroutable or rejected message fails the block.
|
|
126
|
+
Delivery is at-least-once; a failed block raises PublishError and is not
|
|
127
|
+
retried.
|
|
128
|
+
"""
|
|
129
|
+
exchange, routing_key = _destination(destination)
|
|
130
|
+
_bounds(batch_size, max_message_bytes, confirm)
|
|
131
|
+
blocks = _blocks(_rows(batch), batch_size, max_message_bytes)
|
|
132
|
+
if confirm:
|
|
133
|
+
try:
|
|
134
|
+
channel.confirm_delivery()
|
|
135
|
+
except Exception:
|
|
136
|
+
raise PublishError("RabbitMQ publisher confirms could not be enabled; inspect the channel.") from None
|
|
137
|
+
count, messages = 0, 0
|
|
138
|
+
for block in blocks:
|
|
139
|
+
try:
|
|
140
|
+
for body, properties in block:
|
|
141
|
+
channel.basic_publish(exchange, routing_key, body, properties=properties)
|
|
142
|
+
except Exception:
|
|
143
|
+
raise PublishError(
|
|
144
|
+
f"RabbitMQ publish block {count + 1} failed; earlier blocks may be delivered. "
|
|
145
|
+
"Inspect the destination and deduplicate by message_id on retry."
|
|
146
|
+
) from None
|
|
147
|
+
count += 1
|
|
148
|
+
messages += len(block)
|
|
149
|
+
return PublishResult(f"{exchange}/{routing_key}", messages, count)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def publish_operation(operation: str, args: dict[str, Any] | None, channel: Any,
|
|
153
|
+
destination: tuple[str, str], *, api_key_env: str | None = None,
|
|
154
|
+
complete_history: bool = True, max_pages: int = 10000,
|
|
155
|
+
client_factory: Any = None, batch_size: int = 1000,
|
|
156
|
+
max_message_bytes: int = MAX_MESSAGE_BYTES, confirm: bool = False) -> PublishResult:
|
|
157
|
+
"""Read one public operation and publish its complete response envelope."""
|
|
158
|
+
_destination(destination)
|
|
159
|
+
_bounds(batch_size, max_message_bytes, confirm)
|
|
160
|
+
batch = read(operation, args, api_key_env=api_key_env,
|
|
161
|
+
complete_history=complete_history, max_pages=max_pages,
|
|
162
|
+
client_factory=client_factory)
|
|
163
|
+
return publish(batch, channel, destination, batch_size=batch_size,
|
|
164
|
+
max_message_bytes=max_message_bytes, confirm=confirm)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
__all__ = ["COLUMNS", "HEADERS", "MAX_MESSAGE_BYTES", "PublishError", "PublishResult",
|
|
168
|
+
"encode_message", "message_key", "publish", "publish_operation"]
|
|
File without changes
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rabbitmq-fxmacrodata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Publish public FXMacroData responses to RabbitMQ
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=package
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: fxmacrodata-platform-core==0.1.0
|
|
11
|
+
Requires-Dist: pika<2,>=1.4.4
|
|
12
|
+
Provides-Extra: test
|
|
13
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# FXMacroData for RabbitMQ
|
|
17
|
+
|
|
18
|
+
Publish FXMacroData public responses to a RabbitMQ exchange with the official
|
|
19
|
+
`pika` Python client. Route macro announcements to your own queues while
|
|
20
|
+
retaining the source payload and revision metadata.
|
|
21
|
+
|
|
22
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=readme)
|
|
23
|
+
|
|
24
|
+
## Install and connect
|
|
25
|
+
|
|
26
|
+
Requires Python 3.10 or later. Install this package and its adjacent public
|
|
27
|
+
client dependencies from the distribution supplied to you:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/rabbitmq-fxmacrodata
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Configure `RABBITMQ_URL` (an `amqps://` URL with the credentials your
|
|
34
|
+
administrator provides) in your runtime secret store. Then publish to a
|
|
35
|
+
customer-owned exchange:
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import os
|
|
39
|
+
import pika
|
|
40
|
+
from fxmacrodata_platform_core import read
|
|
41
|
+
from fxmacrodata_rabbitmq import publish
|
|
42
|
+
|
|
43
|
+
connection = pika.BlockingConnection(pika.URLParameters(os.environ["RABBITMQ_URL"]))
|
|
44
|
+
try:
|
|
45
|
+
channel = connection.channel()
|
|
46
|
+
batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
|
|
47
|
+
result = publish(batch, channel, ("fxmacrodata", "usd.policy_rate"), confirm=True)
|
|
48
|
+
print(result.messages, "messages in", result.batches, "blocks")
|
|
49
|
+
finally:
|
|
50
|
+
connection.close()
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The exchange and any bound queues must already exist. The destination is an
|
|
54
|
+
`(exchange, routing_key)` tuple; names accept letters, digits, `_`, `.`, `:`
|
|
55
|
+
and `-` up to 255 characters. Use `("", "queue-name")` for the default
|
|
56
|
+
exchange and `("exchange-name", "")` for a fanout exchange. Wildcards, URLs
|
|
57
|
+
and other values are rejected before any publish.
|
|
58
|
+
|
|
59
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is
|
|
60
|
+
available, including REST and MCP operations. `publish_operation` reads one
|
|
61
|
+
operation and publishes it in a single call:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from fxmacrodata_rabbitmq import publish_operation
|
|
65
|
+
|
|
66
|
+
publish_operation("release_calendar", {"currency": "usd"}, channel, ("fxmacrodata", "usd.calendar"),
|
|
67
|
+
api_key_env="FXMD_API_KEY")
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
## Message format
|
|
71
|
+
|
|
72
|
+
Each envelope row becomes one persistent message (`delivery_mode=2`). The
|
|
73
|
+
body is canonical UTF-8 JSON with the keys `operation`, `source_url`,
|
|
74
|
+
`record_id`, `currency`, `indicator`, `date`, `announcement_datetime`,
|
|
75
|
+
`value`, `value_text`, `record_json`, `response_json` and the publisher's
|
|
76
|
+
`content_hash` (SHA-256 of the ten source columns). Only `value` is numeric.
|
|
77
|
+
Dates and announcement times remain source strings. No publication time is
|
|
78
|
+
inferred, and no missing value is filled. `record_json` retains the original
|
|
79
|
+
record; `response_json` retains the complete response, including pagination
|
|
80
|
+
and metadata. An empty response produces one message with `record_json`
|
|
81
|
+
equal to `"null"`, so absence of data and the server's metadata remain
|
|
82
|
+
inspectable. It is not an observation.
|
|
83
|
+
|
|
84
|
+
`message_id` is `operation:record_id:content_hash`; the same source record
|
|
85
|
+
with the same content always produces the same id, and changed data or
|
|
86
|
+
metadata produces a different one. `content_type` is `application/json` and
|
|
87
|
+
the headers `operation`, `record_id`, `content_hash` and `source_url` are set
|
|
88
|
+
on every message. No timestamp property is derived from the data.
|
|
89
|
+
|
|
90
|
+
## Delivery guarantee
|
|
91
|
+
|
|
92
|
+
Delivery is at-least-once. Messages are published in blocks of `batch_size`
|
|
93
|
+
rows (1 to 10,000, 1,000 by default). With `confirm=True` the channel enters
|
|
94
|
+
publisher-confirm mode before the first publish, so a message the broker
|
|
95
|
+
rejects or cannot route fails its block; without confirms, a broker-side
|
|
96
|
+
failure surfaces only when the channel closes. A failed block raises
|
|
97
|
+
`PublishError` without the broker's own diagnostic text and is not retried;
|
|
98
|
+
earlier blocks may already be delivered. Consumers should deduplicate on
|
|
99
|
+
`message_id`, or on the `operation`, `record_id` and `content_hash` headers.
|
|
100
|
+
|
|
101
|
+
`max_message_bytes` (16 MiB by default) rejects an individual envelope larger
|
|
102
|
+
than the budget before anything is published; keep it within the broker's
|
|
103
|
+
`max_message_size`, and narrow the source date range to reduce the response
|
|
104
|
+
size. The source response is materialized before publishing; choose a source
|
|
105
|
+
date range suitable for memory.
|
|
106
|
+
|
|
107
|
+
`publish_operation` accepts `api_key_env`, `complete_history`, `max_pages`
|
|
108
|
+
and `client_factory` for the public source reader. Supply source credentials
|
|
109
|
+
through your runtime secret store, exposing only the environment variable
|
|
110
|
+
name; omit `api_key_env` for anonymous USD access. Destination authentication
|
|
111
|
+
belongs to the connection you open with the official client. The adapter
|
|
112
|
+
neither saves nor prints credentials and never places one in a message or
|
|
113
|
+
header.
|
|
114
|
+
|
|
115
|
+
The package licence covers this adapter code. FXMacroData data access and
|
|
116
|
+
onward redistribution remain subject to your subscription and applicable
|
|
117
|
+
licence. Publishing data to a broker does not grant permission to publish or
|
|
118
|
+
resell it.
|
|
119
|
+
|
|
120
|
+
## Development
|
|
121
|
+
|
|
122
|
+
Tests run offline against an in-memory fake channel:
|
|
123
|
+
|
|
124
|
+
```sh
|
|
125
|
+
python -m pytest tests -q
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Official SDK reference:
|
|
129
|
+
[pika BlockingChannel.basic_publish](https://pika.readthedocs.io/en/stable/modules/adapters/blocking.html?utm_source=pypi&utm_medium=referral&utm_campaign=rabbitmq-fxmacrodata&utm_content=upstream-reference#pika.adapters.blocking_connection.BlockingChannel.basic_publish).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
PUBLIC_FILES.json
|
|
4
|
+
README.md
|
|
5
|
+
pyproject.toml
|
|
6
|
+
examples/consumer_dedupe.py
|
|
7
|
+
src/fxmacrodata_rabbitmq/__init__.py
|
|
8
|
+
src/fxmacrodata_rabbitmq/py.typed
|
|
9
|
+
src/rabbitmq_fxmacrodata.egg-info/PKG-INFO
|
|
10
|
+
src/rabbitmq_fxmacrodata.egg-info/SOURCES.txt
|
|
11
|
+
src/rabbitmq_fxmacrodata.egg-info/dependency_links.txt
|
|
12
|
+
src/rabbitmq_fxmacrodata.egg-info/requires.txt
|
|
13
|
+
src/rabbitmq_fxmacrodata.egg-info/top_level.txt
|
|
14
|
+
tests/test_publisher.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fxmacrodata_rabbitmq
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import pika
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from fxmacrodata_platform_core import Batch, IntegrationError
|
|
7
|
+
import fxmacrodata_rabbitmq as adapter
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Channel:
|
|
11
|
+
def __init__(self, fail_message=None, fail_confirm=False):
|
|
12
|
+
self.published = []
|
|
13
|
+
self.confirms = 0
|
|
14
|
+
self.fail_message = fail_message
|
|
15
|
+
self.fail_confirm = fail_confirm
|
|
16
|
+
|
|
17
|
+
def confirm_delivery(self):
|
|
18
|
+
if self.fail_confirm:
|
|
19
|
+
raise RuntimeError("private destination diagnostic")
|
|
20
|
+
self.confirms += 1
|
|
21
|
+
|
|
22
|
+
def basic_publish(self, exchange, routing_key, body, properties=None, mandatory=False):
|
|
23
|
+
self.published.append((exchange, routing_key, body, properties))
|
|
24
|
+
if len(self.published) == self.fail_message:
|
|
25
|
+
raise RuntimeError("private destination diagnostic")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
DESTINATION = ("fxmacrodata", "usd.policy_rate")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def batch(count=3, metadata=None):
|
|
32
|
+
return Batch("indicator_history", {
|
|
33
|
+
"data": [{"currency": "USD", "indicator": "policy_rate", "value": str(i),
|
|
34
|
+
"date": "2026-10-01", "publication_time_status": "unknown"}
|
|
35
|
+
for i in range(count)],
|
|
36
|
+
"metadata": metadata or {"revision": 1},
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def test_messages_preserve_full_payload_and_carry_dedup_properties():
|
|
41
|
+
source, channel = batch(), Channel()
|
|
42
|
+
result = adapter.publish(source, channel, DESTINATION, batch_size=2)
|
|
43
|
+
assert result == adapter.PublishResult("fxmacrodata/usd.policy_rate", 3, 2)
|
|
44
|
+
exchange, routing_key, body, properties = channel.published[0]
|
|
45
|
+
row = json.loads(body.decode("utf-8"))
|
|
46
|
+
assert (exchange, routing_key) == DESTINATION
|
|
47
|
+
assert set(row) == set(adapter.COLUMNS)
|
|
48
|
+
assert row["date"] == "2026-10-01" and row["announcement_datetime"] is None
|
|
49
|
+
assert row["value"] == 0.0 and row["value_text"] == "0"
|
|
50
|
+
assert json.loads(row["record_json"])["publication_time_status"] == "unknown"
|
|
51
|
+
assert json.loads(row["response_json"]) == source.payload
|
|
52
|
+
assert isinstance(properties, pika.BasicProperties)
|
|
53
|
+
assert properties.delivery_mode == 2 and properties.content_type == "application/json"
|
|
54
|
+
assert properties.message_id == f"indicator_history:{row['record_id']}:{row['content_hash']}"
|
|
55
|
+
assert properties.headers == {"operation": "indicator_history", "record_id": row["record_id"],
|
|
56
|
+
"content_hash": row["content_hash"], "source_url": "https://fxmacrodata.com"}
|
|
57
|
+
assert body == json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()
|
|
58
|
+
assert channel.confirms == 0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_empty_response_is_a_metadata_envelope_not_an_observation():
|
|
62
|
+
source, channel = batch(0), Channel()
|
|
63
|
+
result = adapter.publish(source, channel, DESTINATION)
|
|
64
|
+
row = json.loads(channel.published[0][2])
|
|
65
|
+
assert result.messages == 1 and row["record_json"] == "null"
|
|
66
|
+
assert row["value"] is None and row["date"] is None
|
|
67
|
+
assert json.loads(row["response_json"]) == source.payload
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_retries_have_stable_keys_and_changed_metadata_is_distinct():
|
|
71
|
+
first, retry, changed = Channel(), Channel(), Channel()
|
|
72
|
+
adapter.publish(batch(1), first, DESTINATION)
|
|
73
|
+
adapter.publish(batch(1), retry, DESTINATION)
|
|
74
|
+
adapter.publish(batch(1, {"revision": 2}), changed, DESTINATION)
|
|
75
|
+
assert first.published[0][2] == retry.published[0][2]
|
|
76
|
+
assert first.published[0][3].message_id == retry.published[0][3].message_id
|
|
77
|
+
assert first.published[0][3].headers["record_id"] == changed.published[0][3].headers["record_id"]
|
|
78
|
+
assert first.published[0][3].message_id != changed.published[0][3].message_id
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@pytest.mark.parametrize("destination", [
|
|
82
|
+
"fxmacrodata", ("fxmacrodata",), ("", ""), ("amq.topic", "usd.*"), ("ex change", "k"),
|
|
83
|
+
("amqp://host/ex", "k"), ("ex", "x" * 256), (None, "k"), ("ex", 7),
|
|
84
|
+
])
|
|
85
|
+
def test_bad_destinations_never_publish(destination):
|
|
86
|
+
channel = Channel()
|
|
87
|
+
with pytest.raises(ValueError):
|
|
88
|
+
adapter.publish(batch(), channel, destination)
|
|
89
|
+
assert channel.published == [] and channel.confirms == 0
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_default_exchange_and_fanout_routing_are_accepted():
|
|
93
|
+
channel = Channel()
|
|
94
|
+
adapter.publish(batch(1), channel, ("", "fxmacrodata-queue"))
|
|
95
|
+
adapter.publish(batch(1), channel, ("fxmacrodata-fanout", ""))
|
|
96
|
+
assert [(e, k) for e, k, _, _ in channel.published] == [("", "fxmacrodata-queue"), ("fxmacrodata-fanout", "")]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@pytest.mark.parametrize("options", [
|
|
100
|
+
{"batch_size": 0}, {"batch_size": -1}, {"batch_size": 10001}, {"batch_size": True},
|
|
101
|
+
{"batch_size": 2.5}, {"max_message_bytes": 0}, {"max_message_bytes": 16 * 1024 * 1024 + 1},
|
|
102
|
+
{"confirm": "yes"},
|
|
103
|
+
])
|
|
104
|
+
def test_batch_bounds(options):
|
|
105
|
+
channel = Channel()
|
|
106
|
+
with pytest.raises(ValueError):
|
|
107
|
+
adapter.publish(batch(), channel, DESTINATION, **options)
|
|
108
|
+
assert channel.published == [] and channel.confirms == 0
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def test_failed_block_is_not_retried_or_exposed():
|
|
112
|
+
channel = Channel(fail_message=2)
|
|
113
|
+
with pytest.raises(adapter.PublishError) as error:
|
|
114
|
+
adapter.publish(batch(), channel, DESTINATION, batch_size=1)
|
|
115
|
+
assert "block 2" in str(error.value)
|
|
116
|
+
assert "private destination diagnostic" not in str(error.value)
|
|
117
|
+
assert len(channel.published) == 2
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def test_oversized_envelope_fails_before_any_publish():
|
|
121
|
+
channel = Channel()
|
|
122
|
+
with pytest.raises(ValueError, match="narrow the source date range"):
|
|
123
|
+
adapter.publish(batch(), channel, DESTINATION, max_message_bytes=1, confirm=True)
|
|
124
|
+
assert channel.published == [] and channel.confirms == 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_confirm_enables_publisher_confirms_before_the_first_publish():
|
|
128
|
+
channel = Channel()
|
|
129
|
+
adapter.publish(batch(2), channel, DESTINATION, confirm=True)
|
|
130
|
+
assert channel.confirms == 1 and len(channel.published) == 2
|
|
131
|
+
failing = Channel(fail_confirm=True)
|
|
132
|
+
with pytest.raises(adapter.PublishError) as error:
|
|
133
|
+
adapter.publish(batch(2), failing, DESTINATION, confirm=True)
|
|
134
|
+
assert "private destination diagnostic" not in str(error.value)
|
|
135
|
+
assert failing.published == []
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_operation_forwards_public_reader_options(monkeypatch):
|
|
139
|
+
calls = []
|
|
140
|
+
monkeypatch.setattr(adapter, "read", lambda *args, **kw: calls.append((args, kw)) or batch(1))
|
|
141
|
+
factory = object()
|
|
142
|
+
adapter.publish_operation("indicator_history", {"currency": "usd", "indicator": "policy_rate"},
|
|
143
|
+
Channel(), DESTINATION, api_key_env="FXMD_API_KEY", max_pages=12,
|
|
144
|
+
complete_history=False, client_factory=factory)
|
|
145
|
+
assert calls[0][0] == ("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
|
|
146
|
+
assert calls[0][1] == dict(api_key_env="FXMD_API_KEY", complete_history=False,
|
|
147
|
+
max_pages=12, client_factory=factory)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def test_invalid_configuration_never_reads_source(monkeypatch):
|
|
151
|
+
def unexpected(*args, **kwargs):
|
|
152
|
+
pytest.fail("source must not be called for invalid publisher configuration")
|
|
153
|
+
monkeypatch.setattr(adapter, "read", unexpected)
|
|
154
|
+
for options in ({"batch_size": 0}, {"max_message_bytes": 0}, {"confirm": 1}):
|
|
155
|
+
with pytest.raises(ValueError):
|
|
156
|
+
adapter.publish_operation("release_calendar", {"currency": "usd"}, Channel(), DESTINATION, **options)
|
|
157
|
+
with pytest.raises(ValueError):
|
|
158
|
+
adapter.publish_operation("release_calendar", {"currency": "usd"}, Channel(), ("", ""))
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_credential_arguments_are_rejected_before_publishing():
|
|
162
|
+
channel = Channel()
|
|
163
|
+
with pytest.raises(IntegrationError):
|
|
164
|
+
adapter.publish_operation("indicator_history", {"currency": "usd", "indicator": "policy_rate",
|
|
165
|
+
"api_key": "secret"}, channel, DESTINATION)
|
|
166
|
+
with pytest.raises(IntegrationError):
|
|
167
|
+
adapter.publish_operation("release_calendar", {"currency": "usd"}, channel, DESTINATION,
|
|
168
|
+
api_key_env="not-an-env-name")
|
|
169
|
+
assert channel.published == []
|