dnaStreaming 2.0.5__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dnaStreaming-2.0.5/dnaStreaming.egg-info → dnastreaming-2.1.0}/PKG-INFO +34 -18
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/README.rst +24 -13
- dnastreaming-2.1.0/dnaStreaming/config.py +109 -0
- dnastreaming-2.1.0/dnaStreaming/customer_config.json +5 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/listener.py +2 -3
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/services/credentials_service.py +1 -1
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/test/test_config.py +0 -50
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0/dnaStreaming.egg-info}/PKG-INFO +34 -18
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/setup.py +1 -1
- dnaStreaming-2.0.5/dnaStreaming/config.py +0 -202
- dnaStreaming-2.0.5/dnaStreaming/customer_config.json +0 -8
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/LICENSE +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/MANIFEST.in +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/__init__.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/demo/__init__.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/demo/pubsub_stream.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/demo/show_stream.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/demo/show_stream_async.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/services/__init__.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/services/authentication_service.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/services/pubsub_service.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming/test/__init__.py +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming.egg-info/SOURCES.txt +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming.egg-info/dependency_links.txt +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming.egg-info/requires.txt +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/dnaStreaming.egg-info/top_level.txt +0 -0
- {dnaStreaming-2.0.5 → dnastreaming-2.1.0}/setup.cfg +0 -0
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dnaStreaming
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Dow Jones DNA Streaming Project
|
|
5
5
|
Home-page: https://github.com/dowjones/dj-dna-streams-python
|
|
6
|
-
Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.0.
|
|
6
|
+
Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.1.0.tar.gz
|
|
7
7
|
Author: Zachary Kagan
|
|
8
8
|
Author-email: zachary.kagan@dowjones.com
|
|
9
9
|
License: MIT
|
|
10
10
|
Keywords: DOWJONES,FACTIVA,STREAMS
|
|
11
|
-
Platform: UNKNOWN
|
|
12
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
13
12
|
Classifier: Intended Audience :: Developers
|
|
14
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
14
|
Classifier: Programming Language :: Python :: 3
|
|
16
15
|
Description-Content-Type: text/x-rst
|
|
17
16
|
License-File: LICENSE
|
|
17
|
+
Requires-Dist: googleapis-common-protos>=1.56.4
|
|
18
|
+
Requires-Dist: google-auth>=2.11.0
|
|
19
|
+
Requires-Dist: google-cloud-pubsub==2.13.6
|
|
20
|
+
Requires-Dist: google-cloud-core==2.3.2
|
|
21
|
+
Requires-Dist: google-api-core==2.10.1
|
|
22
|
+
Requires-Dist: mock>=3.0.5
|
|
23
|
+
Requires-Dist: oauth2client>=4.1.3
|
|
24
|
+
Requires-Dist: requests>=2.28.0
|
|
18
25
|
|
|
19
26
|
How To Use
|
|
20
27
|
----------
|
|
@@ -40,14 +47,17 @@ There is currently one way to authenticate, which is by using **your user key**.
|
|
|
40
47
|
Configuring
|
|
41
48
|
___________
|
|
42
49
|
|
|
43
|
-
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this
|
|
50
|
+
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this: you can set environment variables, use a configuration file or pass an argument to a constructor.
|
|
44
51
|
|
|
45
52
|
1. Set environment variables.
|
|
46
53
|
###################################################################
|
|
47
54
|
|
|
48
|
-
To set your service account credentials, set:
|
|
55
|
+
To set your service account credentials, set the `USER_KEY` environment variable:
|
|
56
|
+
|
|
57
|
+
.. code-block::
|
|
58
|
+
|
|
59
|
+
export USER_KEY="1234567890-098765432"
|
|
49
60
|
|
|
50
|
-
- An environment variable named 'USER_KEY'.
|
|
51
61
|
|
|
52
62
|
To set your subscription ID, simply set an environment variable named 'SUBSCRIPTION_ID' like so
|
|
53
63
|
|
|
@@ -61,18 +71,33 @@ The code above is the command line expression for setting this environment varia
|
|
|
61
71
|
2. Using the configuration file.
|
|
62
72
|
###################################################################
|
|
63
73
|
|
|
64
|
-
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this
|
|
74
|
+
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this option, fill the JSON object within by adding your user key and your subscription ID. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
|
|
75
|
+
|
|
76
|
+
> The listener will search for the `customer_config.json` file inside your `$HOME` directory by default.
|
|
77
|
+
|
|
78
|
+
If you prefer using an explicit path to your configuration file, pass the absolute path to the Listener constructor like so:
|
|
79
|
+
|
|
80
|
+
.. code-block:: python
|
|
81
|
+
|
|
82
|
+
from dnaStreaming.listener import Listener
|
|
83
|
+
# Config. file authentication
|
|
84
|
+
listener = Listener(config_file=<ABSOLUTE PATH TO YOUR CONFIG. FILE>)
|
|
85
|
+
|
|
65
86
|
|
|
66
87
|
3. Pass in variables as function arguments.
|
|
67
88
|
###################################################################
|
|
68
89
|
|
|
69
|
-
You may pass your
|
|
90
|
+
You may pass your user key to the Listener constructor and your subscription ID to the listen method like so:
|
|
70
91
|
|
|
71
92
|
.. code-block:: python
|
|
72
93
|
|
|
73
94
|
from dnaStreaming.listener import Listener
|
|
74
|
-
#
|
|
95
|
+
# Use the user_key argument to provide your credentials
|
|
75
96
|
listener = Listener(user_key=<YOUR USER KEY>)
|
|
97
|
+
# Use the subscription_id argument to provide your subscription id to the listener
|
|
98
|
+
listener.listen(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
99
|
+
# same parameter for the async variation
|
|
100
|
+
listener.listen_async(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
76
101
|
|
|
77
102
|
|
|
78
103
|
Or you may use the environment variables.
|
|
@@ -218,13 +243,6 @@ Or
|
|
|
218
243
|
python ./dnaStreaming/demo/show_stream_async.py -s
|
|
219
244
|
|
|
220
245
|
|
|
221
|
-
If you are having `ImportError: No module named ...` run this in your terminal before running the demo:
|
|
222
|
-
|
|
223
|
-
.. code-block::
|
|
224
|
-
|
|
225
|
-
export PYTHONPATH='.'
|
|
226
|
-
|
|
227
|
-
|
|
228
246
|
Running Docker Demo
|
|
229
247
|
|
|
230
248
|
Execute the following at the project root:
|
|
@@ -235,5 +253,3 @@ Execute the following at the project root:
|
|
|
235
253
|
-e USER_KEY=<your user KEY> \
|
|
236
254
|
-e SUBSCRIPTION_ID=<your subscription ID> \
|
|
237
255
|
dj-dna-streaming-python
|
|
238
|
-
|
|
239
|
-
|
|
@@ -22,14 +22,17 @@ There is currently one way to authenticate, which is by using **your user key**.
|
|
|
22
22
|
Configuring
|
|
23
23
|
___________
|
|
24
24
|
|
|
25
|
-
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this
|
|
25
|
+
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this: you can set environment variables, use a configuration file or pass an argument to a constructor.
|
|
26
26
|
|
|
27
27
|
1. Set environment variables.
|
|
28
28
|
###################################################################
|
|
29
29
|
|
|
30
|
-
To set your service account credentials, set:
|
|
30
|
+
To set your service account credentials, set the `USER_KEY` environment variable:
|
|
31
|
+
|
|
32
|
+
.. code-block::
|
|
33
|
+
|
|
34
|
+
export USER_KEY="1234567890-098765432"
|
|
31
35
|
|
|
32
|
-
- An environment variable named 'USER_KEY'.
|
|
33
36
|
|
|
34
37
|
To set your subscription ID, simply set an environment variable named 'SUBSCRIPTION_ID' like so
|
|
35
38
|
|
|
@@ -43,18 +46,33 @@ The code above is the command line expression for setting this environment varia
|
|
|
43
46
|
2. Using the configuration file.
|
|
44
47
|
###################################################################
|
|
45
48
|
|
|
46
|
-
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this
|
|
49
|
+
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this option, fill the JSON object within by adding your user key and your subscription ID. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
|
|
50
|
+
|
|
51
|
+
> The listener will search for the `customer_config.json` file inside your `$HOME` directory by default.
|
|
52
|
+
|
|
53
|
+
If you prefer using an explicit path to your configuration file, pass the absolute path to the Listener constructor like so:
|
|
54
|
+
|
|
55
|
+
.. code-block:: python
|
|
56
|
+
|
|
57
|
+
from dnaStreaming.listener import Listener
|
|
58
|
+
# Config. file authentication
|
|
59
|
+
listener = Listener(config_file=<ABSOLUTE PATH TO YOUR CONFIG. FILE>)
|
|
60
|
+
|
|
47
61
|
|
|
48
62
|
3. Pass in variables as function arguments.
|
|
49
63
|
###################################################################
|
|
50
64
|
|
|
51
|
-
You may pass your
|
|
65
|
+
You may pass your user key to the Listener constructor and your subscription ID to the listen method like so:
|
|
52
66
|
|
|
53
67
|
.. code-block:: python
|
|
54
68
|
|
|
55
69
|
from dnaStreaming.listener import Listener
|
|
56
|
-
#
|
|
70
|
+
# Use the user_key argument to provide your credentials
|
|
57
71
|
listener = Listener(user_key=<YOUR USER KEY>)
|
|
72
|
+
# Use the subscription_id argument to provide your subscription id to the listener
|
|
73
|
+
listener.listen(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
74
|
+
# same parameter for the async variation
|
|
75
|
+
listener.listen_async(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
58
76
|
|
|
59
77
|
|
|
60
78
|
Or you may use the environment variables.
|
|
@@ -200,13 +218,6 @@ Or
|
|
|
200
218
|
python ./dnaStreaming/demo/show_stream_async.py -s
|
|
201
219
|
|
|
202
220
|
|
|
203
|
-
If you are having `ImportError: No module named ...` run this in your terminal before running the demo:
|
|
204
|
-
|
|
205
|
-
.. code-block::
|
|
206
|
-
|
|
207
|
-
export PYTHONPATH='.'
|
|
208
|
-
|
|
209
|
-
|
|
210
221
|
Running Docker Demo
|
|
211
222
|
|
|
212
223
|
Execute the following at the project root:
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import absolute_import, division
|
|
2
|
+
|
|
3
|
+
import errno
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
class Config(object):
|
|
9
|
+
|
|
10
|
+
DEFAULT_HOST = 'https://api.dowjones.com'
|
|
11
|
+
|
|
12
|
+
DEFAULT_CUST_CONFIG_PATH = str(Path.home()) + '/customer_config.json'
|
|
13
|
+
ENV_VAR_SUBSCRIPTION_ID = 'SUBSCRIPTION_ID'
|
|
14
|
+
ENV_VAR_USER_KEY = 'USER_KEY'
|
|
15
|
+
ENV_VAR_SERVICE_ACCOUNT_ID = 'SERVICE_ACCOUNT_ID'
|
|
16
|
+
ENV_VAR_API_HOST = 'API_HOST'
|
|
17
|
+
|
|
18
|
+
def __init__(self, service_account_id=None, user_key=None, config_file=None):
|
|
19
|
+
self.customer_config_path = self.DEFAULT_CUST_CONFIG_PATH if config_file is None else config_file
|
|
20
|
+
self.initialized = False
|
|
21
|
+
self.service_account_id = service_account_id
|
|
22
|
+
self.user_key = user_key
|
|
23
|
+
|
|
24
|
+
self.headers = None
|
|
25
|
+
|
|
26
|
+
def _initialize(self):
|
|
27
|
+
self._validate()
|
|
28
|
+
|
|
29
|
+
with open(self.customer_config_path, 'r') as f:
|
|
30
|
+
self.customer_config = json.load(f)
|
|
31
|
+
|
|
32
|
+
self.initialized = True
|
|
33
|
+
self.headers = None
|
|
34
|
+
|
|
35
|
+
def _validate(self):
|
|
36
|
+
if not os.path.isfile(self.customer_config_path):
|
|
37
|
+
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.customer_config_path)
|
|
38
|
+
|
|
39
|
+
if not os.access(self.customer_config_path, os.R_OK):
|
|
40
|
+
raise Exception('Encountered permission problem reading file from path \'{}\'.'.format(self.customer_config_path))
|
|
41
|
+
|
|
42
|
+
def get_headers(self):
|
|
43
|
+
if self.headers:
|
|
44
|
+
return self.headers
|
|
45
|
+
else:
|
|
46
|
+
self.headers = self.get_authentication_headers()
|
|
47
|
+
return self.headers
|
|
48
|
+
|
|
49
|
+
def get_authentication_headers(self):
|
|
50
|
+
|
|
51
|
+
user_key = self.get_user_key()
|
|
52
|
+
if user_key:
|
|
53
|
+
return {
|
|
54
|
+
'user-key': user_key
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
else:
|
|
58
|
+
msg = """
|
|
59
|
+
Unable to find credentials. Specify your account credentials by choosing one of the following ways:
|
|
60
|
+
- set your user key to an env var under the name 'USER_KEY'
|
|
61
|
+
- place your customer_config.json file in your home directory (e.g. ~/, $HOME/)
|
|
62
|
+
- pass the absolute path of your customer_config.json file to the constructor of the Listener class
|
|
63
|
+
- pass the user key as a parameter to the constructor of the Listener class
|
|
64
|
+
"""
|
|
65
|
+
raise Exception(msg)
|
|
66
|
+
|
|
67
|
+
def get_uri_context(self):
|
|
68
|
+
host = os.getenv(self.ENV_VAR_API_HOST, self.DEFAULT_HOST)
|
|
69
|
+
return host
|
|
70
|
+
|
|
71
|
+
# in the following two methods, note that we use "SERVICE_ACCOUNT_ID" as a legacy,
|
|
72
|
+
# alternate name for the "USER_KEY" parameter, from the customer's perspective
|
|
73
|
+
def get_user_key(self):
|
|
74
|
+
user_key = self.user_key if self.user_key else self.service_account_id
|
|
75
|
+
|
|
76
|
+
if user_key is None:
|
|
77
|
+
user_key = os.getenv(self.ENV_VAR_USER_KEY,
|
|
78
|
+
os.getenv(self.ENV_VAR_SERVICE_ACCOUNT_ID))
|
|
79
|
+
|
|
80
|
+
if user_key is None:
|
|
81
|
+
user_key = self._user_key_id_from_file()
|
|
82
|
+
|
|
83
|
+
return user_key
|
|
84
|
+
|
|
85
|
+
def _user_key_id_from_file(self):
|
|
86
|
+
if not self.initialized:
|
|
87
|
+
self._initialize()
|
|
88
|
+
|
|
89
|
+
return self.customer_config.get('user_key',
|
|
90
|
+
self.customer_config.get('service_account_id'))
|
|
91
|
+
|
|
92
|
+
def subscription(self):
|
|
93
|
+
if os.getenv(self.ENV_VAR_SUBSCRIPTION_ID) is not None:
|
|
94
|
+
subscription = os.getenv(self.ENV_VAR_SUBSCRIPTION_ID)
|
|
95
|
+
else:
|
|
96
|
+
subscription = self._subscription_id_from_file()
|
|
97
|
+
|
|
98
|
+
return subscription
|
|
99
|
+
|
|
100
|
+
def _set_customer_config_path(self, path):
|
|
101
|
+
self.customer_config_path = path
|
|
102
|
+
self._initialize()
|
|
103
|
+
|
|
104
|
+
def _subscription_id_from_file(self):
|
|
105
|
+
if not self.initialized:
|
|
106
|
+
self._initialize()
|
|
107
|
+
|
|
108
|
+
return self.customer_config['subscription_id']
|
|
109
|
+
|
|
@@ -14,9 +14,8 @@ from dnaStreaming.services import pubsub_service, credentials_service
|
|
|
14
14
|
class Listener(object):
|
|
15
15
|
DEFAULT_UNLIMITED_MESSAGES = None
|
|
16
16
|
|
|
17
|
-
def __init__(self, service_account_id=None, user_key=None,
|
|
18
|
-
config = Config(service_account_id, user_key,
|
|
19
|
-
user_id, client_id, password)
|
|
17
|
+
def __init__(self, service_account_id=None, user_key=None, config_file=None):
|
|
18
|
+
config = Config(service_account_id, user_key, config_file)
|
|
20
19
|
self._initialize(config)
|
|
21
20
|
self.current_subscription_index = 0
|
|
22
21
|
|
|
@@ -5,7 +5,7 @@ import requests
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
def fetch_credentials(config):
|
|
8
|
-
response = _get_requests().get(config.get_uri_context() + '/accounts/streaming-credentials',
|
|
8
|
+
response = _get_requests().get(config.get_uri_context() + '/sns-accounts/streaming-credentials',
|
|
9
9
|
headers=config.get_headers())
|
|
10
10
|
|
|
11
11
|
if response.status_code == 401:
|
|
@@ -13,9 +13,6 @@ class TestConfig(TestCase):
|
|
|
13
13
|
self.ensure_remove_environment_variable(Config.ENV_VAR_USER_KEY)
|
|
14
14
|
self.ensure_remove_environment_variable(Config.ENV_VAR_SUBSCRIPTION_ID)
|
|
15
15
|
self.ensure_remove_environment_variable(Config.ENV_VAR_API_HOST)
|
|
16
|
-
self.ensure_remove_environment_variable(Config.ENV_VAR_USER_ID)
|
|
17
|
-
self.ensure_remove_environment_variable(Config.ENV_VAR_CLIENT_ID)
|
|
18
|
-
self.ensure_remove_environment_variable(Config.ENV_VAR_PASSWORD)
|
|
19
16
|
|
|
20
17
|
def ensure_remove_environment_variable(self, key):
|
|
21
18
|
if key in os.environ:
|
|
@@ -55,22 +52,15 @@ class TestConfig(TestCase):
|
|
|
55
52
|
# Act
|
|
56
53
|
user_key = config.get_user_key()
|
|
57
54
|
subscription = config.subscription()
|
|
58
|
-
oauth2_credentials = config.oauth2_credentials()
|
|
59
55
|
|
|
60
56
|
# Assert
|
|
61
57
|
assert user_key
|
|
62
58
|
assert subscription == 'bar'
|
|
63
|
-
assert oauth2_credentials.get('user_id')
|
|
64
|
-
assert oauth2_credentials.get('password')
|
|
65
|
-
assert oauth2_credentials.get('client_id')
|
|
66
59
|
|
|
67
60
|
def test_environment_variables_success(self):
|
|
68
61
|
# Arrange
|
|
69
62
|
os.environ[Config.ENV_VAR_USER_KEY] = '123'
|
|
70
63
|
os.environ[Config.ENV_VAR_SUBSCRIPTION_ID] = 'ABC'
|
|
71
|
-
os.environ[Config.ENV_VAR_USER_ID] = 'user'
|
|
72
|
-
os.environ[Config.ENV_VAR_CLIENT_ID] = 'client'
|
|
73
|
-
os.environ[Config.ENV_VAR_PASSWORD] = 'password'
|
|
74
64
|
|
|
75
65
|
# Act
|
|
76
66
|
config = Config()
|
|
@@ -82,9 +72,6 @@ class TestConfig(TestCase):
|
|
|
82
72
|
assert os.environ[Config.ENV_VAR_USER_KEY] == config.get_user_key()
|
|
83
73
|
subscription_id = config.subscription()
|
|
84
74
|
assert subscription_id == 'ABC'
|
|
85
|
-
assert os.environ[Config.ENV_VAR_USER_ID] == config.oauth2_credentials().get('user_id')
|
|
86
|
-
assert os.environ[Config.ENV_VAR_CLIENT_ID] == config.oauth2_credentials().get('client_id')
|
|
87
|
-
assert os.environ[Config.ENV_VAR_PASSWORD] == config.oauth2_credentials().get('password')
|
|
88
75
|
|
|
89
76
|
def test_environment_variable_service_account_id_success(self):
|
|
90
77
|
# Arrange
|
|
@@ -128,42 +115,5 @@ class TestConfig(TestCase):
|
|
|
128
115
|
# Assert
|
|
129
116
|
assert config.get_user_key() == '123'
|
|
130
117
|
|
|
131
|
-
@patch.object(Config, '_fetch_jwt', return_value='test_jwt_value')
|
|
132
|
-
def test_get_headers_jwt(self, fetch_jwt_mock):
|
|
133
|
-
# Arrange
|
|
134
|
-
config = Config()
|
|
135
|
-
|
|
136
|
-
fileFolder = os.path.dirname(os.path.realpath(__file__))
|
|
137
|
-
config._set_customer_config_path(os.path.join(fileFolder, 'test_customer_config.json'))
|
|
138
|
-
|
|
139
|
-
headers_expected = {
|
|
140
|
-
'Authorization': 'test_jwt_value'
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
# Act
|
|
144
|
-
headers_actual = config.get_authentication_headers()
|
|
145
|
-
|
|
146
|
-
# Assert
|
|
147
|
-
assert headers_actual == headers_expected
|
|
148
|
-
fetch_jwt_mock.assert_called_once()
|
|
149
|
-
|
|
150
|
-
@patch.object(Config, '_fetch_jwt', return_value='test_jwt_value')
|
|
151
|
-
def test_get_headers_user_key(self, fetch_jwt_mock):
|
|
152
|
-
# Arrange
|
|
153
|
-
user_key = "just some user key"
|
|
154
|
-
config = Config(user_key)
|
|
155
|
-
|
|
156
|
-
headers_expected = {
|
|
157
|
-
'user-key': user_key
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
# Act
|
|
161
|
-
headers_actual = config.get_authentication_headers()
|
|
162
|
-
|
|
163
|
-
# Assert
|
|
164
|
-
assert headers_actual == headers_expected
|
|
165
|
-
fetch_jwt_mock.assert_not_called()
|
|
166
|
-
|
|
167
|
-
|
|
168
118
|
if __name__ == '__main__' and __package__ is None:
|
|
169
119
|
unittest.main()
|
|
@@ -1,20 +1,27 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: dnaStreaming
|
|
3
|
-
Version: 2.0
|
|
3
|
+
Version: 2.1.0
|
|
4
4
|
Summary: Dow Jones DNA Streaming Project
|
|
5
5
|
Home-page: https://github.com/dowjones/dj-dna-streams-python
|
|
6
|
-
Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.0.
|
|
6
|
+
Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.1.0.tar.gz
|
|
7
7
|
Author: Zachary Kagan
|
|
8
8
|
Author-email: zachary.kagan@dowjones.com
|
|
9
9
|
License: MIT
|
|
10
10
|
Keywords: DOWJONES,FACTIVA,STREAMS
|
|
11
|
-
Platform: UNKNOWN
|
|
12
11
|
Classifier: Development Status :: 5 - Production/Stable
|
|
13
12
|
Classifier: Intended Audience :: Developers
|
|
14
13
|
Classifier: License :: OSI Approved :: MIT License
|
|
15
14
|
Classifier: Programming Language :: Python :: 3
|
|
16
15
|
Description-Content-Type: text/x-rst
|
|
17
16
|
License-File: LICENSE
|
|
17
|
+
Requires-Dist: googleapis-common-protos>=1.56.4
|
|
18
|
+
Requires-Dist: google-auth>=2.11.0
|
|
19
|
+
Requires-Dist: google-cloud-pubsub==2.13.6
|
|
20
|
+
Requires-Dist: google-cloud-core==2.3.2
|
|
21
|
+
Requires-Dist: google-api-core==2.10.1
|
|
22
|
+
Requires-Dist: mock>=3.0.5
|
|
23
|
+
Requires-Dist: oauth2client>=4.1.3
|
|
24
|
+
Requires-Dist: requests>=2.28.0
|
|
18
25
|
|
|
19
26
|
How To Use
|
|
20
27
|
----------
|
|
@@ -40,14 +47,17 @@ There is currently one way to authenticate, which is by using **your user key**.
|
|
|
40
47
|
Configuring
|
|
41
48
|
___________
|
|
42
49
|
|
|
43
|
-
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this
|
|
50
|
+
To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this: you can set environment variables, use a configuration file or pass an argument to a constructor.
|
|
44
51
|
|
|
45
52
|
1. Set environment variables.
|
|
46
53
|
###################################################################
|
|
47
54
|
|
|
48
|
-
To set your service account credentials, set:
|
|
55
|
+
To set your service account credentials, set the `USER_KEY` environment variable:
|
|
56
|
+
|
|
57
|
+
.. code-block::
|
|
58
|
+
|
|
59
|
+
export USER_KEY="1234567890-098765432"
|
|
49
60
|
|
|
50
|
-
- An environment variable named 'USER_KEY'.
|
|
51
61
|
|
|
52
62
|
To set your subscription ID, simply set an environment variable named 'SUBSCRIPTION_ID' like so
|
|
53
63
|
|
|
@@ -61,18 +71,33 @@ The code above is the command line expression for setting this environment varia
|
|
|
61
71
|
2. Using the configuration file.
|
|
62
72
|
###################################################################
|
|
63
73
|
|
|
64
|
-
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this
|
|
74
|
+
In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this option, fill the JSON object within by adding your user key and your subscription ID. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
|
|
75
|
+
|
|
76
|
+
> The listener will search for the `customer_config.json` file inside your `$HOME` directory by default.
|
|
77
|
+
|
|
78
|
+
If you prefer using an explicit path to your configuration file, pass the absolute path to the Listener constructor like so:
|
|
79
|
+
|
|
80
|
+
.. code-block:: python
|
|
81
|
+
|
|
82
|
+
from dnaStreaming.listener import Listener
|
|
83
|
+
# Config. file authentication
|
|
84
|
+
listener = Listener(config_file=<ABSOLUTE PATH TO YOUR CONFIG. FILE>)
|
|
85
|
+
|
|
65
86
|
|
|
66
87
|
3. Pass in variables as function arguments.
|
|
67
88
|
###################################################################
|
|
68
89
|
|
|
69
|
-
You may pass your
|
|
90
|
+
You may pass your user key to the Listener constructor and your subscription ID to the listen method like so:
|
|
70
91
|
|
|
71
92
|
.. code-block:: python
|
|
72
93
|
|
|
73
94
|
from dnaStreaming.listener import Listener
|
|
74
|
-
#
|
|
95
|
+
# Use the user_key argument to provide your credentials
|
|
75
96
|
listener = Listener(user_key=<YOUR USER KEY>)
|
|
97
|
+
# Use the subscription_id argument to provide your subscription id to the listener
|
|
98
|
+
listener.listen(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
99
|
+
# same parameter for the async variation
|
|
100
|
+
listener.listen_async(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
|
|
76
101
|
|
|
77
102
|
|
|
78
103
|
Or you may use the environment variables.
|
|
@@ -218,13 +243,6 @@ Or
|
|
|
218
243
|
python ./dnaStreaming/demo/show_stream_async.py -s
|
|
219
244
|
|
|
220
245
|
|
|
221
|
-
If you are having `ImportError: No module named ...` run this in your terminal before running the demo:
|
|
222
|
-
|
|
223
|
-
.. code-block::
|
|
224
|
-
|
|
225
|
-
export PYTHONPATH='.'
|
|
226
|
-
|
|
227
|
-
|
|
228
246
|
Running Docker Demo
|
|
229
247
|
|
|
230
248
|
Execute the following at the project root:
|
|
@@ -235,5 +253,3 @@ Execute the following at the project root:
|
|
|
235
253
|
-e USER_KEY=<your user KEY> \
|
|
236
254
|
-e SUBSCRIPTION_ID=<your subscription ID> \
|
|
237
255
|
dj-dna-streaming-python
|
|
238
|
-
|
|
239
|
-
|
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
from __future__ import absolute_import, division
|
|
2
|
-
|
|
3
|
-
import errno
|
|
4
|
-
import json
|
|
5
|
-
import os
|
|
6
|
-
import requests
|
|
7
|
-
|
|
8
|
-
class Config(object):
|
|
9
|
-
OAUTH_URL = 'https://accounts.dowjones.com/oauth2/v1/token'
|
|
10
|
-
DEFAULT_HOST = 'https://api.dowjones.com'
|
|
11
|
-
|
|
12
|
-
DEFAULT_CUST_CONFIG_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), './customer_config.json'))
|
|
13
|
-
ENV_VAR_SUBSCRIPTION_ID = 'SUBSCRIPTION_ID'
|
|
14
|
-
ENV_VAR_USER_KEY = 'USER_KEY'
|
|
15
|
-
ENV_VAR_SERVICE_ACCOUNT_ID = 'SERVICE_ACCOUNT_ID'
|
|
16
|
-
ENV_VAR_USER_ID = 'USER_ID'
|
|
17
|
-
ENV_VAR_CLIENT_ID = 'CLIENT_ID'
|
|
18
|
-
ENV_VAR_PASSWORD = 'PASSWORD'
|
|
19
|
-
ENV_VAR_API_HOST = 'API_HOST'
|
|
20
|
-
|
|
21
|
-
def __init__(self, service_account_id=None, user_key=None, user_id=None, client_id=None, password=None):
|
|
22
|
-
self.customer_config_path = self.DEFAULT_CUST_CONFIG_PATH
|
|
23
|
-
self.initialized = False
|
|
24
|
-
self.service_account_id = service_account_id
|
|
25
|
-
self.user_key = user_key
|
|
26
|
-
self.user_id = user_id
|
|
27
|
-
self.client_id = client_id
|
|
28
|
-
self.password = password
|
|
29
|
-
|
|
30
|
-
self.headers = None
|
|
31
|
-
|
|
32
|
-
def _initialize(self):
|
|
33
|
-
self._validate()
|
|
34
|
-
|
|
35
|
-
with open(self.customer_config_path, 'r') as f:
|
|
36
|
-
self.customer_config = json.load(f)
|
|
37
|
-
|
|
38
|
-
self.initialized = True
|
|
39
|
-
self.headers = None
|
|
40
|
-
|
|
41
|
-
def _validate(self):
|
|
42
|
-
if not os.path.isfile(self.customer_config_path):
|
|
43
|
-
raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.customer_config_path)
|
|
44
|
-
|
|
45
|
-
if not os.access(self.customer_config_path, os.R_OK):
|
|
46
|
-
raise Exception('Encountered permission problem reading file from path \'{}\'.'.format(self.customer_config_path))
|
|
47
|
-
|
|
48
|
-
def get_headers(self):
|
|
49
|
-
if self.headers:
|
|
50
|
-
return self.headers
|
|
51
|
-
else:
|
|
52
|
-
self.headers = self.get_authentication_headers()
|
|
53
|
-
return self.headers
|
|
54
|
-
|
|
55
|
-
def get_authentication_headers(self):
|
|
56
|
-
if self.oauth2_credentials():
|
|
57
|
-
return {
|
|
58
|
-
'Authorization': self._fetch_jwt()
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
# missing oauth creds, authenticate the old way via user key
|
|
62
|
-
user_key = self.get_user_key()
|
|
63
|
-
if user_key:
|
|
64
|
-
return {
|
|
65
|
-
'user-key': user_key
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
else:
|
|
69
|
-
msg = '''Could not find determine credentials:
|
|
70
|
-
Must specify account credentials as user_id, client_id, and password, either through env vars, customer_config.json, or as args to Listener constructor
|
|
71
|
-
(see README.rst)'''
|
|
72
|
-
raise Exception(msg)
|
|
73
|
-
|
|
74
|
-
def _fetch_jwt(self):
|
|
75
|
-
oauth2_credentials = self.oauth2_credentials()
|
|
76
|
-
user_id = oauth2_credentials.get('user_id')
|
|
77
|
-
client_id = oauth2_credentials.get('client_id')
|
|
78
|
-
password = oauth2_credentials.get('password')
|
|
79
|
-
|
|
80
|
-
# two requests need to be made, to the same URL, with slightly different params, to finally obtain a JWT
|
|
81
|
-
# the second request contains params returned in the response of the first request
|
|
82
|
-
# I know this makes no sense but it is what it is
|
|
83
|
-
body = {
|
|
84
|
-
'username': user_id,
|
|
85
|
-
'client_id': client_id,
|
|
86
|
-
'password': password,
|
|
87
|
-
'connection': 'service-account',
|
|
88
|
-
'grant_type': 'password',
|
|
89
|
-
'scope': 'openid service_account_id'
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
try:
|
|
93
|
-
|
|
94
|
-
response = _get_requests().post(self.OAUTH_URL, data=body).json()
|
|
95
|
-
body['scope'] = 'openid pib'
|
|
96
|
-
body['grant_type'] = 'urn:ietf:params:oauth:grant-type:jwt-bearer'
|
|
97
|
-
body['access_token'] = response.get('access_token')
|
|
98
|
-
body['assertion'] = response.get('id_token')
|
|
99
|
-
|
|
100
|
-
response = _get_requests().post(self.OAUTH_URL, data=body).json()
|
|
101
|
-
|
|
102
|
-
return '{0} {1}'.format(response['token_type'], response['access_token'])
|
|
103
|
-
except (KeyError, ValueError):
|
|
104
|
-
msg = '''Unable to retrieve JWT with the given credentials:
|
|
105
|
-
User ID: {0}
|
|
106
|
-
Client ID: {1}
|
|
107
|
-
Password: {2}
|
|
108
|
-
'''.format(user_id, client_id, password)
|
|
109
|
-
raise Exception(msg)
|
|
110
|
-
|
|
111
|
-
def get_uri_context(self):
|
|
112
|
-
headers = self.get_headers()
|
|
113
|
-
host = os.getenv(self.ENV_VAR_API_HOST, self.DEFAULT_HOST)
|
|
114
|
-
if "Authorization" in headers:
|
|
115
|
-
return host + '/dna'
|
|
116
|
-
elif 'user-key' in headers:
|
|
117
|
-
return host + '/alpha'
|
|
118
|
-
else:
|
|
119
|
-
msg = '''Could not determine user credentials:
|
|
120
|
-
Must specify account credentials as user_id, client_id, and password, either through env vars, customer_config.json, or as args to Listener constructor
|
|
121
|
-
(see README.rst)'''
|
|
122
|
-
raise Exception(msg)
|
|
123
|
-
|
|
124
|
-
# return credentials (user_id, client_id, and password) for obtaining a JWT via OAuth2 if all these fields are defined in the constructor, env vars or config file
|
|
125
|
-
# otherwise return None (the client will have to authenticate API request with an user key, i.e. the original standard)
|
|
126
|
-
def oauth2_credentials(self):
|
|
127
|
-
creds = self._build_oauth2_credentials(
|
|
128
|
-
self.user_id,
|
|
129
|
-
self.client_id,
|
|
130
|
-
self.password
|
|
131
|
-
)
|
|
132
|
-
if not creds:
|
|
133
|
-
creds = self._build_oauth2_credentials(
|
|
134
|
-
os.getenv(self.ENV_VAR_USER_ID),
|
|
135
|
-
os.getenv(self.ENV_VAR_CLIENT_ID),
|
|
136
|
-
os.getenv(self.ENV_VAR_PASSWORD)
|
|
137
|
-
)
|
|
138
|
-
if not creds:
|
|
139
|
-
creds = self._oauth2_credentials_from_file()
|
|
140
|
-
return creds
|
|
141
|
-
|
|
142
|
-
def _oauth2_credentials_from_file(self):
|
|
143
|
-
if not self.initialized:
|
|
144
|
-
self._initialize()
|
|
145
|
-
|
|
146
|
-
return self._build_oauth2_credentials(
|
|
147
|
-
self.customer_config.get('user_id'),
|
|
148
|
-
self.customer_config.get('client_id'),
|
|
149
|
-
self.customer_config.get('password')
|
|
150
|
-
)
|
|
151
|
-
|
|
152
|
-
def _build_oauth2_credentials(self, user_id, client_id, password):
|
|
153
|
-
if user_id and client_id and password:
|
|
154
|
-
return {
|
|
155
|
-
'user_id': user_id,
|
|
156
|
-
'client_id': client_id,
|
|
157
|
-
'password': password
|
|
158
|
-
}
|
|
159
|
-
return None
|
|
160
|
-
|
|
161
|
-
# in the following two methods, note that we use "SERVICE_ACCOUNT_ID" as a legacy,
|
|
162
|
-
# alternate name for the "USER_KEY" parameter, from the customer's perspective
|
|
163
|
-
def get_user_key(self):
|
|
164
|
-
user_key = self.user_key if self.user_key else self.service_account_id
|
|
165
|
-
|
|
166
|
-
if user_key is None:
|
|
167
|
-
user_key = os.getenv(self.ENV_VAR_USER_KEY,
|
|
168
|
-
os.getenv(self.ENV_VAR_SERVICE_ACCOUNT_ID))
|
|
169
|
-
|
|
170
|
-
if user_key is None:
|
|
171
|
-
user_key = self._user_key_id_from_file()
|
|
172
|
-
|
|
173
|
-
return user_key
|
|
174
|
-
|
|
175
|
-
def _user_key_id_from_file(self):
|
|
176
|
-
if not self.initialized:
|
|
177
|
-
self._initialize()
|
|
178
|
-
|
|
179
|
-
return self.customer_config.get('user_key',
|
|
180
|
-
self.customer_config.get('service_account_id'))
|
|
181
|
-
|
|
182
|
-
def subscription(self):
|
|
183
|
-
if os.getenv(self.ENV_VAR_SUBSCRIPTION_ID) is not None:
|
|
184
|
-
subscription = os.getenv(self.ENV_VAR_SUBSCRIPTION_ID)
|
|
185
|
-
else:
|
|
186
|
-
subscription = self._subscription_id_from_file()
|
|
187
|
-
|
|
188
|
-
return subscription
|
|
189
|
-
|
|
190
|
-
def _set_customer_config_path(self, path):
|
|
191
|
-
self.customer_config_path = path
|
|
192
|
-
self._initialize()
|
|
193
|
-
|
|
194
|
-
def _subscription_id_from_file(self):
|
|
195
|
-
if not self.initialized:
|
|
196
|
-
self._initialize()
|
|
197
|
-
|
|
198
|
-
return self.customer_config['subscription_id']
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
def _get_requests():
|
|
202
|
-
return requests
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|