dnaStreaming 2.4.0__tar.gz → 2.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/CHANGELOG.md +6 -1
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/PKG-INFO +1 -1
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/listener.py +28 -11
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/credentials_service.py +9 -2
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/pubsub_service.py +5 -2
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming.egg-info/PKG-INFO +1 -1
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming.egg-info/SOURCES.txt +3 -0
- dnastreaming-2.4.1/src/dnaStreaming.egg-info/scm_file_list.json +27 -0
- dnastreaming-2.4.1/src/dnaStreaming.egg-info/scm_version.json +8 -0
- dnastreaming-2.4.1/tests/test_credentials_service.py +141 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/.github/workflows/publish.yaml +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/.gitignore +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/LICENSE +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/README.md +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/demo/__init__.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/demo/pubsub_stream.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/demo/show_ha_stream_async.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/demo/show_stream.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/demo/show_stream_async.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/pyproject.toml +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/setup.cfg +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/__init__.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/config.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/customer_config.json +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/__init__.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/authentication_service.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/availability_service.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming.egg-info/dependency_links.txt +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming.egg-info/requires.txt +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming.egg-info/top_level.txt +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/tests/test_async.py +0 -0
- {dnastreaming-2.4.0 → dnastreaming-2.4.1}/tests/test_sync.py +0 -0
|
@@ -149,4 +149,9 @@ the subscription *and* consuming it. Instead they will just consume an already e
|
|
|
149
149
|
- [added] - Authentication via OAuth bearer token
|
|
150
150
|
- [changed] - Upgraded to Python 3.14
|
|
151
151
|
- [changed] - Upgraded dependencies
|
|
152
|
-
- [changed] - Upgraded package to work with pyproject.toml
|
|
152
|
+
- [changed] - Upgraded package to work with pyproject.toml
|
|
153
|
+
|
|
154
|
+
2.4.1 / 2026-10-22
|
|
155
|
+
==================
|
|
156
|
+
- [changed] - Requests to the streaming credentials endpoint now include the `X-Subscription-Id` header with the subscription ID being listened to
|
|
157
|
+
- [fixed] - Streaming credentials are now fetched once per listen call instead of once per Pub/Sub client plus once for the project ID (previously 2 requests for `listen`/`listen_async` and 3 for `listen_async_ha`)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dnaStreaming
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.1
|
|
4
4
|
Summary: A Python 3 library for Dow Jones customers to consume data from a Factiva Analytics Stream.
|
|
5
5
|
Author-email: Zachary Kagan <zachary.kagan@dowjones.com>, Daniel Morales <daniel.morales@dowjones.com>, Alan Fuentes <alan.fuentes@dowjones.com>, Carlos Blazquez <carlos.blazquez@dowjones.com>, Carlos Niño <carlos.ninoalarcon@dowjones.com>
|
|
6
6
|
License: MIT
|
|
@@ -48,8 +48,6 @@ class Listener(object):
|
|
|
48
48
|
subscription_id="",
|
|
49
49
|
batch_size=10,
|
|
50
50
|
):
|
|
51
|
-
pubsub_client = pubsub_service.get_client(self.config)
|
|
52
|
-
|
|
53
51
|
subscription_id = subscription_id or self.config.subscription()
|
|
54
52
|
if not subscription_id:
|
|
55
53
|
raise Exception(
|
|
@@ -57,7 +55,12 @@ class Listener(object):
|
|
|
57
55
|
+ "variable, a config file or by passing the value to the method."
|
|
58
56
|
)
|
|
59
57
|
|
|
60
|
-
streaming_credentials = credentials_service.fetch_credentials(
|
|
58
|
+
streaming_credentials = credentials_service.fetch_credentials(
|
|
59
|
+
self.config, subscription_id
|
|
60
|
+
)
|
|
61
|
+
pubsub_client = pubsub_service.get_client(
|
|
62
|
+
self.config, subscription_id, streaming_credentials=streaming_credentials
|
|
63
|
+
)
|
|
61
64
|
subscription_path = pubsub_client.subscription_path(
|
|
62
65
|
streaming_credentials["project_id"], subscription_id
|
|
63
66
|
)
|
|
@@ -119,7 +122,7 @@ class Listener(object):
|
|
|
119
122
|
"the stream again."
|
|
120
123
|
)
|
|
121
124
|
time.sleep(10)
|
|
122
|
-
pubsub_client = pubsub_service.get_client(self.config)
|
|
125
|
+
pubsub_client = pubsub_service.get_client(self.config, subscription_id)
|
|
123
126
|
|
|
124
127
|
def listen_async(self, on_message_callback, subscription_id=""):
|
|
125
128
|
def ack_message_and_callback(message):
|
|
@@ -131,8 +134,6 @@ class Listener(object):
|
|
|
131
134
|
on_message_callback(news_msg, subscription_id)
|
|
132
135
|
message.ack()
|
|
133
136
|
|
|
134
|
-
pubsub_client = pubsub_service.get_client(self.config)
|
|
135
|
-
|
|
136
137
|
subscription_id = subscription_id or self.config.subscription()
|
|
137
138
|
|
|
138
139
|
if not subscription_id:
|
|
@@ -141,7 +142,12 @@ class Listener(object):
|
|
|
141
142
|
"by passing the value to the method."
|
|
142
143
|
)
|
|
143
144
|
|
|
144
|
-
streaming_credentials = credentials_service.fetch_credentials(
|
|
145
|
+
streaming_credentials = credentials_service.fetch_credentials(
|
|
146
|
+
self.config, subscription_id
|
|
147
|
+
)
|
|
148
|
+
pubsub_client = pubsub_service.get_client(
|
|
149
|
+
self.config, subscription_id, streaming_credentials=streaming_credentials
|
|
150
|
+
)
|
|
145
151
|
subscription_path = pubsub_client.subscription_path(
|
|
146
152
|
streaming_credentials["project_id"], subscription_id
|
|
147
153
|
)
|
|
@@ -164,9 +170,6 @@ class Listener(object):
|
|
|
164
170
|
on_message_callback(news_msg, short_subscription_id)
|
|
165
171
|
message.ack()
|
|
166
172
|
|
|
167
|
-
main_pubsub_client = pubsub_service.get_client(self.config, MAIN_REGION)
|
|
168
|
-
backup_pubsub_client = pubsub_service.get_client(self.config, BACKUP_REGION)
|
|
169
|
-
|
|
170
173
|
subscription_id = self.config.subscription()
|
|
171
174
|
|
|
172
175
|
if not subscription_id:
|
|
@@ -175,7 +178,21 @@ class Listener(object):
|
|
|
175
178
|
"by passing the value to the method."
|
|
176
179
|
)
|
|
177
180
|
|
|
178
|
-
streaming_credentials = credentials_service.fetch_credentials(
|
|
181
|
+
streaming_credentials = credentials_service.fetch_credentials(
|
|
182
|
+
self.config, subscription_id
|
|
183
|
+
)
|
|
184
|
+
main_pubsub_client = pubsub_service.get_client(
|
|
185
|
+
self.config,
|
|
186
|
+
subscription_id,
|
|
187
|
+
region=MAIN_REGION,
|
|
188
|
+
streaming_credentials=streaming_credentials,
|
|
189
|
+
)
|
|
190
|
+
backup_pubsub_client = pubsub_service.get_client(
|
|
191
|
+
self.config,
|
|
192
|
+
subscription_id,
|
|
193
|
+
region=BACKUP_REGION,
|
|
194
|
+
streaming_credentials=streaming_credentials,
|
|
195
|
+
)
|
|
179
196
|
|
|
180
197
|
api_host = self.config.get_uri_context()
|
|
181
198
|
user_key = self.config.get_user_key()
|
|
@@ -4,11 +4,18 @@ import json
|
|
|
4
4
|
|
|
5
5
|
import requests
|
|
6
6
|
|
|
7
|
+
SUBSCRIPTION_ID_HEADER = "X-Subscription-Id"
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def fetch_credentials(config, subscription_id=None):
|
|
11
|
+
# Copy the cached auth headers so the per-subscription header is not persisted in the config.
|
|
12
|
+
headers = dict(config.get_headers())
|
|
13
|
+
if subscription_id:
|
|
14
|
+
headers[SUBSCRIPTION_ID_HEADER] = subscription_id
|
|
7
15
|
|
|
8
|
-
def fetch_credentials(config):
|
|
9
16
|
response = requests.get(
|
|
10
17
|
config.get_uri_context() + "/sns-accounts/streaming-credentials",
|
|
11
|
-
headers=
|
|
18
|
+
headers=headers,
|
|
12
19
|
)
|
|
13
20
|
|
|
14
21
|
if response.status_code == 401:
|
|
@@ -6,8 +6,11 @@ from dnaStreaming.services import authentication_service, credentials_service
|
|
|
6
6
|
from dnaStreaming.services.availability_service import BACKUP_REGION, MAIN_REGION
|
|
7
7
|
|
|
8
8
|
|
|
9
|
-
def get_client(config, region=None):
|
|
10
|
-
streaming_credentials
|
|
9
|
+
def get_client(config, subscription_id=None, region=None, streaming_credentials=None):
|
|
10
|
+
if streaming_credentials is None:
|
|
11
|
+
streaming_credentials = credentials_service.fetch_credentials(
|
|
12
|
+
config, subscription_id
|
|
13
|
+
)
|
|
11
14
|
credentials = authentication_service.get_authenticated_oauth_credentials(
|
|
12
15
|
streaming_credentials
|
|
13
16
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: dnaStreaming
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.1
|
|
4
4
|
Summary: A Python 3 library for Dow Jones customers to consume data from a Factiva Analytics Stream.
|
|
5
5
|
Author-email: Zachary Kagan <zachary.kagan@dowjones.com>, Daniel Morales <daniel.morales@dowjones.com>, Alan Fuentes <alan.fuentes@dowjones.com>, Carlos Blazquez <carlos.blazquez@dowjones.com>, Carlos Niño <carlos.ninoalarcon@dowjones.com>
|
|
6
6
|
License: MIT
|
|
@@ -17,6 +17,8 @@ src/dnaStreaming.egg-info/PKG-INFO
|
|
|
17
17
|
src/dnaStreaming.egg-info/SOURCES.txt
|
|
18
18
|
src/dnaStreaming.egg-info/dependency_links.txt
|
|
19
19
|
src/dnaStreaming.egg-info/requires.txt
|
|
20
|
+
src/dnaStreaming.egg-info/scm_file_list.json
|
|
21
|
+
src/dnaStreaming.egg-info/scm_version.json
|
|
20
22
|
src/dnaStreaming.egg-info/top_level.txt
|
|
21
23
|
src/dnaStreaming/services/__init__.py
|
|
22
24
|
src/dnaStreaming/services/authentication_service.py
|
|
@@ -24,4 +26,5 @@ src/dnaStreaming/services/availability_service.py
|
|
|
24
26
|
src/dnaStreaming/services/credentials_service.py
|
|
25
27
|
src/dnaStreaming/services/pubsub_service.py
|
|
26
28
|
tests/test_async.py
|
|
29
|
+
tests/test_credentials_service.py
|
|
27
30
|
tests/test_sync.py
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
{
|
|
2
|
+
"files": [
|
|
3
|
+
".github/workflows/publish.yaml",
|
|
4
|
+
".gitignore",
|
|
5
|
+
"CHANGELOG.md",
|
|
6
|
+
"LICENSE",
|
|
7
|
+
"README.md",
|
|
8
|
+
"demo/__init__.py",
|
|
9
|
+
"demo/pubsub_stream.py",
|
|
10
|
+
"demo/show_ha_stream_async.py",
|
|
11
|
+
"demo/show_stream.py",
|
|
12
|
+
"demo/show_stream_async.py",
|
|
13
|
+
"pyproject.toml",
|
|
14
|
+
"src/dnaStreaming/__init__.py",
|
|
15
|
+
"src/dnaStreaming/config.py",
|
|
16
|
+
"src/dnaStreaming/customer_config.json",
|
|
17
|
+
"src/dnaStreaming/listener.py",
|
|
18
|
+
"src/dnaStreaming/services/__init__.py",
|
|
19
|
+
"src/dnaStreaming/services/authentication_service.py",
|
|
20
|
+
"src/dnaStreaming/services/availability_service.py",
|
|
21
|
+
"src/dnaStreaming/services/credentials_service.py",
|
|
22
|
+
"src/dnaStreaming/services/pubsub_service.py",
|
|
23
|
+
"tests/test_async.py",
|
|
24
|
+
"tests/test_credentials_service.py",
|
|
25
|
+
"tests/test_sync.py"
|
|
26
|
+
]
|
|
27
|
+
}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from unittest.mock import MagicMock, patch
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from dnaStreaming.config import Config
|
|
7
|
+
from dnaStreaming.services import credentials_service, pubsub_service
|
|
8
|
+
from dnaStreaming.services.availability_service import MAIN_REGION
|
|
9
|
+
from dnaStreaming.services.credentials_service import SUBSCRIPTION_ID_HEADER
|
|
10
|
+
|
|
11
|
+
STREAMING_CREDENTIALS = {"project_id": "p"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@pytest.fixture(autouse=True)
|
|
15
|
+
def clean_env(monkeypatch):
|
|
16
|
+
for var in (
|
|
17
|
+
Config.ENV_VAR_OAUTH_TOKEN,
|
|
18
|
+
Config.ENV_VAR_USER_KEY,
|
|
19
|
+
Config.ENV_VAR_SERVICE_ACCOUNT_ID,
|
|
20
|
+
Config.ENV_VAR_API_HOST,
|
|
21
|
+
):
|
|
22
|
+
monkeypatch.delenv(var, raising=False)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@pytest.fixture
|
|
26
|
+
def config():
|
|
27
|
+
return Config(oauth_token="tok")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@pytest.fixture
|
|
31
|
+
def mock_get():
|
|
32
|
+
response = MagicMock()
|
|
33
|
+
response.status_code = 200
|
|
34
|
+
response.text = json.dumps(
|
|
35
|
+
{
|
|
36
|
+
"data": {
|
|
37
|
+
"attributes": {
|
|
38
|
+
"streaming_credentials": json.dumps(STREAMING_CREDENTIALS)
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
)
|
|
43
|
+
with patch.object(
|
|
44
|
+
credentials_service.requests, "get", return_value=response
|
|
45
|
+
) as mocked:
|
|
46
|
+
yield mocked
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_fetch_credentials_sends_subscription_id_header(config, mock_get):
|
|
50
|
+
result = credentials_service.fetch_credentials(config, "sub-123")
|
|
51
|
+
|
|
52
|
+
url = mock_get.call_args.args[0]
|
|
53
|
+
headers = mock_get.call_args.kwargs["headers"]
|
|
54
|
+
assert url.endswith("/sns-accounts/streaming-credentials")
|
|
55
|
+
assert headers["Authorization"] == "Bearer tok"
|
|
56
|
+
assert headers[SUBSCRIPTION_ID_HEADER] == "sub-123"
|
|
57
|
+
assert result == STREAMING_CREDENTIALS
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def test_fetch_credentials_does_not_mutate_cached_headers(config, mock_get):
|
|
61
|
+
credentials_service.fetch_credentials(config, "sub-123")
|
|
62
|
+
assert SUBSCRIPTION_ID_HEADER not in config.get_headers()
|
|
63
|
+
|
|
64
|
+
credentials_service.fetch_credentials(config, "sub-456")
|
|
65
|
+
headers = mock_get.call_args.kwargs["headers"]
|
|
66
|
+
assert headers[SUBSCRIPTION_ID_HEADER] == "sub-456"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_fetch_credentials_without_subscription_id_omits_header(config, mock_get):
|
|
70
|
+
credentials_service.fetch_credentials(config)
|
|
71
|
+
|
|
72
|
+
headers = mock_get.call_args.kwargs["headers"]
|
|
73
|
+
assert SUBSCRIPTION_ID_HEADER not in headers
|
|
74
|
+
assert headers["Authorization"] == "Bearer tok"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_get_client_passes_subscription_id(config):
|
|
78
|
+
with (
|
|
79
|
+
patch.object(
|
|
80
|
+
pubsub_service.credentials_service,
|
|
81
|
+
"fetch_credentials",
|
|
82
|
+
return_value=STREAMING_CREDENTIALS,
|
|
83
|
+
) as mock_fetch,
|
|
84
|
+
patch.object(
|
|
85
|
+
pubsub_service.authentication_service,
|
|
86
|
+
"get_authenticated_oauth_credentials",
|
|
87
|
+
),
|
|
88
|
+
patch.object(pubsub_service, "SubscriberClient") as mock_client,
|
|
89
|
+
):
|
|
90
|
+
pubsub_service.get_client(config, "sub-123", region=MAIN_REGION)
|
|
91
|
+
|
|
92
|
+
mock_fetch.assert_called_once_with(config, "sub-123")
|
|
93
|
+
assert mock_client.call_args.kwargs["client_options"] == {
|
|
94
|
+
"api_endpoint": f"{MAIN_REGION}-pubsub.googleapis.com:443"
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_get_client_uses_provided_credentials_without_fetching(config):
|
|
99
|
+
with (
|
|
100
|
+
patch.object(
|
|
101
|
+
pubsub_service.credentials_service, "fetch_credentials"
|
|
102
|
+
) as mock_fetch,
|
|
103
|
+
patch.object(
|
|
104
|
+
pubsub_service.authentication_service,
|
|
105
|
+
"get_authenticated_oauth_credentials",
|
|
106
|
+
) as mock_auth,
|
|
107
|
+
patch.object(pubsub_service, "SubscriberClient"),
|
|
108
|
+
):
|
|
109
|
+
pubsub_service.get_client(
|
|
110
|
+
config, "sub-123", streaming_credentials=STREAMING_CREDENTIALS
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
mock_fetch.assert_not_called()
|
|
114
|
+
mock_auth.assert_called_once_with(STREAMING_CREDENTIALS)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@pytest.mark.parametrize("method", ["listen", "listen_async", "listen_async_ha"])
|
|
118
|
+
def test_listener_fetches_credentials_once(config, mock_get, monkeypatch, method):
|
|
119
|
+
from dnaStreaming import listener as listener_module
|
|
120
|
+
from dnaStreaming.listener import Listener
|
|
121
|
+
|
|
122
|
+
monkeypatch.setattr(config, "subscription", lambda: "sub-123")
|
|
123
|
+
listener = Listener.__new__(Listener)
|
|
124
|
+
listener._initialize(config)
|
|
125
|
+
|
|
126
|
+
with (
|
|
127
|
+
patch.object(
|
|
128
|
+
pubsub_service.authentication_service,
|
|
129
|
+
"get_authenticated_oauth_credentials",
|
|
130
|
+
),
|
|
131
|
+
patch.object(pubsub_service, "SubscriberClient"),
|
|
132
|
+
patch.object(listener_module, "Thread"),
|
|
133
|
+
):
|
|
134
|
+
if method == "listen":
|
|
135
|
+
listener.listen(lambda msg, sub: False, maximum_messages=0)
|
|
136
|
+
else:
|
|
137
|
+
getattr(listener, method)(lambda msg, sub: None)
|
|
138
|
+
|
|
139
|
+
assert mock_get.call_count == 1
|
|
140
|
+
headers = mock_get.call_args.kwargs["headers"]
|
|
141
|
+
assert headers[SUBSCRIPTION_ID_HEADER] == "sub-123"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{dnastreaming-2.4.0 → dnastreaming-2.4.1}/src/dnaStreaming/services/authentication_service.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|