dnaStreaming 2.0.2__tar.gz → 2.1.0__tar.gz

dnastreaming-2.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Dow Jones Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dnastreaming-2.1.0/PKG-INFO

@@ -0,0 +1,255 @@
+Metadata-Version: 2.1
+Name: dnaStreaming
+Version: 2.1.0
+Summary: Dow Jones DNA Streaming Project
+Home-page: https://github.com/dowjones/dj-dna-streams-python
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.1.0.tar.gz
+Author: Zachary Kagan
+Author-email: zachary.kagan@dowjones.com
+License: MIT
+Keywords: DOWJONES,FACTIVA,STREAMS
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Description-Content-Type: text/x-rst
+License-File: LICENSE
+Requires-Dist: googleapis-common-protos>=1.56.4
+Requires-Dist: google-auth>=2.11.0
+Requires-Dist: google-cloud-pubsub==2.13.6
+Requires-Dist: google-cloud-core==2.3.2
+Requires-Dist: google-api-core==2.10.1
+Requires-Dist: mock>=3.0.5
+Requires-Dist: oauth2client>=4.1.3
+Requires-Dist: requests>=2.28.0
+
+How To Use
+----------
+
+This a Python3 library is for Dow Jones customers consuming data from a Dow Jones Factiva Stream.
+
+To use this we recommend 'pip installing' this by making the following addition to your requirements.txt:
+
+.. code-block::
+
+    # To fetch latest version from PyPi
+    dnaStreaming
+
+    # to fetch latest version from GitHub
+    git+https://github.com/dowjones/dj-dna-streams-python#egg=dnaStreaming
+
+
+Auth
+-----------
+
+There is currently one way to authenticate, which is by using **your user key**.
+
+Configuring
+___________
+
+To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this: you can set environment variables, use a configuration file or pass an argument to a constructor.
+
+1. Set environment variables.
+###################################################################
+
+To set your service account credentials, set the `USER_KEY` environment variable:
+
+.. code-block::
+
+    export USER_KEY="1234567890-098765432"
+
+
+To set your subscription ID, simply set an environment variable named 'SUBSCRIPTION_ID' like so
+
+.. code-block::
+
+    export SUBSCRIPTION_ID="ABC1234567889"
+
+
+The code above is the command line expression for setting this environment variable on Mac OSX. Other operating systems might have a slightly different techniques for setting environment variables on the command line.
+
+2. Using the configuration file.
+###################################################################
+
+In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this option, fill the JSON object within by adding your user key and your subscription ID. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
+
+> The listener will search for the `customer_config.json` file inside your `$HOME` directory by default.
+
+If you prefer using an explicit path to your configuration file, pass the absolute path to the Listener constructor like so:
+
+.. code-block:: python
+
+    from dnaStreaming.listener import Listener
+    # Config. file authentication
+    listener = Listener(config_file=<ABSOLUTE PATH TO YOUR CONFIG. FILE>)
+
+
+3. Pass in variables as function arguments.
+###################################################################
+
+You may pass your user key to the Listener constructor and your subscription ID to the listen method like so:
+
+.. code-block:: python
+
+    from dnaStreaming.listener import Listener
+    # Use the user_key argument to provide your credentials
+    listener = Listener(user_key=<YOUR USER KEY>)
+    # Use the subscription_id argument to provide your subscription id to the listener
+    listener.listen(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
+    # same parameter for the async variation
+    listener.listen_async(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
+
+
+Or you may use the environment variables.
+Remember that passing credentials and subscription ID(s) in this way will override the environment variable and the config file settings.
+
+.. code-block:: python
+
+    from dnaStreaming.listener import Listener
+
+    listener = Listener()
+
+
+4. Listening to messages
+###################################################################
+
+You may want to listen messages synchronously like so:
+
+.. code-block:: python
+
+    def callback(message, subscription_id):
+        print('Subscription ID: {}: Message: {}'.format(subscription_id, message.data.__str__()))
+        return True  # If desired return False to stop the message flow. This will unblock the process as well.
+
+    listener.listen(callback, maximum_messages=10)  # Omitting maximum_messages means you will continue to get messages as they appear. Can be a firehose. Use with caution.
+    # You may pass subscription ID as a parameter to the listen function
+
+
+You may want to listen messages asynchronously like so:
+
+.. code-block:: python
+
+    def callback(message, subscription_id):
+        print('Subscription ID: {}: Message: {}'.format(subscription_id, message.data.__str__()))
+
+    future = listener.listen_async(callback)
+    # After calling `listed_async` you need to keep the main thread alive.
+
+    for count in range(0, 5):
+        sleep(1)
+
+    # Stop receiving messages after 5 seconds
+    if future.running():
+        future.cancel()
+
+
+Log Files
+_________
+
+Very minimal logging is written to the module's path 'logs/dj-dna-streaming-python.log'. To keep maintenance simple this log is overwritten every time the app starts.
+
+
+Testing
+_______
+
+.. code-block::
+
+    cd dnaStreaming/test
+    pip install -r requirements.txt
+    py.test . -s
+
+
+or, alternatively, to test against python2.7 and python3.5:
+
+.. code-block::
+
+    tox
+
+
+Flake8
+______
+
+If you are maintaining this library, ensure you run flake8 before you commit. At project root command line:
+
+.. code-block::
+
+    flake8 ./dnaStreaming ./tests
+
+
+Running the Demonstration Code/Development
+__________________________________________
+
+If you are enhancing this codebase (and not just using it as a library), follow these example MacOS steps:
+
+1. Checkout the Project from Git.
+###################################################################
+
+2. Go to the Project Root.
+###################################################################
+
+3. Create a Virtual Environment.
+###################################################################
+
+.. code-block::
+
+    virtualenv venv
+
+
+4. Then activate the virutal environment by executing this command:
+###################################################################
+
+.. code-block::
+
+    source ./venv/bin/activate
+
+
+5. Install the Dependencies
+###################################################################
+
+.. code-block::
+
+    pip install -r requirements.txt
+
+
+6. Install the App:
+###################################################################
+
+.. code-block::
+
+    python setup.py install
+
+
+7. Set the Configuration Variables
+###################################################################
+
+See the config section.
+
+8. Run the Demo Code
+###################################################################
+
+Running Non-Docker Demo:
+
+Execute the following at the project root:
+
+.. code-block::
+
+    python ./dnaStreaming/demo/show_stream.py -s
+
+
+Or
+
+.. code-block::
+
+    python ./dnaStreaming/demo/show_stream_async.py -s
+
+
+Running Docker Demo
+
+Execute the following at the project root:
+
+.. code-block::
+
+    docker run -it \                    
+    -e USER_KEY=<your user KEY> \
+    -e SUBSCRIPTION_ID=<your subscription ID> \
+    dj-dna-streaming-python

{dnaStreaming-2.0.2 → dnastreaming-2.1.0}/README.rst

@@ -7,31 +7,32 @@ To use this we recommend 'pip installing' this by making the following addition
 
 .. code-block::
 
-    git+https://git@github.com/dowjones/dj-dna-streams-python#egg=dj-dna-streams-python
+    # To fetch latest version from PyPi
+    dnaStreaming
+
+    # to fetch latest version from GitHub
+    git+https://github.com/dowjones/dj-dna-streams-python#egg=dnaStreaming
 
 
 Auth
 -----------
 
-There are general two ways to authenticate.
-
-- Your User Key
-
-- UserId, ClientId, and Password
-
+There is currently one way to authenticate, which is by using **your user key**.
 
 Configuring
 ___________
 
-To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this. You can either set environment variables or you can use a configuration file.
+To run this code, you need to provide credentials from one of the authentication methods and your subscriptions. There are 3 ways to do this: you can set environment variables, use a configuration file or pass an argument to a constructor.
 
 1. Set environment variables.
 ###################################################################
 
-To set your service account credentials, set either:
+To set your service account credentials, set the `USER_KEY` environment variable:
+
+.. code-block::
+
+    export USER_KEY="1234567890-098765432"
 
-- An environment variable named 'USER_KEY'.
-- Three environment variable named 'USER_ID', 'CLIENT_ID', and 'PASSWORD'.
 
 To set your subscription ID, simply set an environment variable named 'SUBSCRIPTION_ID' like so
 
@@ -45,21 +46,33 @@ The code above is the command line expression for setting this environment varia
 2. Using the configuration file.
 ###################################################################
 
-In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this configuration file, follow these directions: Open this file and add your service account credentials. Then add your subscription IDs. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
+In this codebase you will find a file named 'customer_config.json'. You are not required to use this file. If you prefer to use this option, fill the JSON object within by adding your user key and your subscription ID. Remember that this is a JSON file so follow basic JSON formatting and syntax conventions.
+
+> The listener will search for the `customer_config.json` file inside your `$HOME` directory by default.
+
+If you prefer using an explicit path to your configuration file, pass the absolute path to the Listener constructor like so:
+
+.. code-block:: python
+
+    from dnaStreaming.listener import Listener
+    # Config. file authentication
+    listener = Listener(config_file=<ABSOLUTE PATH TO YOUR CONFIG. FILE>)
+
 
 3. Pass in variables as function arguments.
 ###################################################################
 
-You may pass your service account credentials (user_id, client_id, and password) to the Listener constructor like so:
+You may pass your user key to the Listener constructor and your subscription ID to the listen method like so:
 
 .. code-block:: python
 
     from dnaStreaming.listener import Listener
-    # User key authentication
+    # Use the user_key argument to provide your credentials
     listener = Listener(user_key=<YOUR USER KEY>)
-
-    # UserId, ClientId and Password authentication
-    listener = Listener(user_id=<YOUR USER ID>, client_id=<YOUR_CLIENT_ID>, password=<YOUR_PASSWORD>)
+    # Use the subscription_id argument to provide your subscription id to the listener
+    listener.listen(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
+    # same parameter for the async variation
+    listener.listen_async(callback, subscription_id=<YOUR SUBSCRIPTION ID>)
 
 
 Or you may use the environment variables.
@@ -205,30 +218,10 @@ Or
     python ./dnaStreaming/demo/show_stream_async.py -s
 
 
-If you are having `ImportError:  No module named ...` run this in your terminal before running the demo:
-
-.. code-block::
-
-    export PYTHONPATH='.'
-
-
 Running Docker Demo
 
 Execute the following at the project root:
 
-.. code-block::
-
-    docker build -f ./DockerfileDemo -t dj-dna-streaming-python .
-
-    docker run -it \
-    -e USER_ID=<your user ID> \
-    -e CLIENT_ID=<your client ID> \
-    -e PASSWORD=<your password> \
-    -e SUBSCRIPTION_ID=<your subscription ID> \
-    dj-dna-streaming-python
-
-or:
-
 .. code-block::
 
     docker run -it \                    

dnastreaming-2.1.0/dnaStreaming/config.py

@@ -0,0 +1,109 @@
+from __future__ import absolute_import, division
+
+import errno
+import json
+import os
+from pathlib import Path
+
+class Config(object):
+
+    DEFAULT_HOST = 'https://api.dowjones.com'
+
+    DEFAULT_CUST_CONFIG_PATH = str(Path.home()) + '/customer_config.json'
+    ENV_VAR_SUBSCRIPTION_ID = 'SUBSCRIPTION_ID'
+    ENV_VAR_USER_KEY = 'USER_KEY'
+    ENV_VAR_SERVICE_ACCOUNT_ID = 'SERVICE_ACCOUNT_ID'
+    ENV_VAR_API_HOST = 'API_HOST'
+
+    def __init__(self, service_account_id=None, user_key=None, config_file=None):
+        self.customer_config_path = self.DEFAULT_CUST_CONFIG_PATH if config_file is None else config_file
+        self.initialized = False
+        self.service_account_id = service_account_id
+        self.user_key = user_key
+
+        self.headers = None
+
+    def _initialize(self):
+        self._validate()
+
+        with open(self.customer_config_path, 'r') as f:
+            self.customer_config = json.load(f)
+
+        self.initialized = True
+        self.headers = None
+
+    def _validate(self):
+        if not os.path.isfile(self.customer_config_path):
+            raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), self.customer_config_path)
+
+        if not os.access(self.customer_config_path, os.R_OK):
+            raise Exception('Encountered permission problem reading file from path \'{}\'.'.format(self.customer_config_path))
+
+    def get_headers(self):
+        if self.headers:
+            return self.headers
+        else:
+            self.headers = self.get_authentication_headers()
+            return self.headers
+
+    def get_authentication_headers(self):
+
+        user_key = self.get_user_key()
+        if user_key:
+            return {
+                'user-key': user_key
+            }
+
+        else:
+            msg = """
+                Unable to find credentials. Specify your account credentials by choosing one of the following ways:
+                - set your user key to an env var under the name 'USER_KEY'
+                - place your customer_config.json file in your home directory (e.g. ~/, $HOME/)
+                - pass the absolute path of your customer_config.json file to the constructor of the Listener class
+                - pass the user key as a parameter to the constructor of the Listener class
+            """
+            raise Exception(msg)
+
+    def get_uri_context(self):
+        host = os.getenv(self.ENV_VAR_API_HOST, self.DEFAULT_HOST)
+        return host
+
+    # in the following two methods, note that we use "SERVICE_ACCOUNT_ID" as a legacy,
+    # alternate name for the "USER_KEY" parameter, from the customer's perspective
+    def get_user_key(self):
+        user_key = self.user_key if self.user_key else self.service_account_id
+
+        if user_key is None:
+            user_key = os.getenv(self.ENV_VAR_USER_KEY,
+                                 os.getenv(self.ENV_VAR_SERVICE_ACCOUNT_ID))
+
+            if user_key is None:
+                user_key = self._user_key_id_from_file()
+
+        return user_key
+
+    def _user_key_id_from_file(self):
+        if not self.initialized:
+            self._initialize()
+
+        return self.customer_config.get('user_key',
+                                        self.customer_config.get('service_account_id'))
+
+    def subscription(self):
+        if os.getenv(self.ENV_VAR_SUBSCRIPTION_ID) is not None:
+            subscription = os.getenv(self.ENV_VAR_SUBSCRIPTION_ID)
+        else:
+            subscription = self._subscription_id_from_file()
+
+        return subscription
+
+    def _set_customer_config_path(self, path):
+        self.customer_config_path = path
+        self._initialize()
+
+    def _subscription_id_from_file(self):
+        if not self.initialized:
+            self._initialize()
+
+        return self.customer_config['subscription_id']
+

dnastreaming-2.1.0/dnaStreaming/customer_config.json

@@ -0,0 +1,5 @@
+{
+
+  "user_key": "",
+  "subscription_id": ""
+}

{dnaStreaming-2.0.2 → dnastreaming-2.1.0}/dnaStreaming/demo/show_stream.py

@@ -6,7 +6,12 @@ quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
 print("\n[ACTIVITY] Receiving messages (SYNC)...\n[0]", end='')
 
 
-def callback(message, subscription_id, file_handle=None):
+def callback(message, subscription_id):
+    # Replace the code below with the custom operations you'd like to apply to each article (message)
+    # like writing to a file, database, or pushing the message to a separate message broker system.
+    # This demo only prints some message details.
+    # Subscription id parameter is provided and can be used for auditing purposes.
+
     callback.counter += 1
     if not quiet_demo:
         if message['action'] != 'del':

dnastreaming-2.1.0/dnaStreaming/listener.py

@@ -0,0 +1,114 @@
+import time
+import requests
+import json
+from threading import Thread
+
+from google.api_core.exceptions import GoogleAPICallError, NotFound
+from google import pubsub_v1
+
+from dnaStreaming import logger
+from dnaStreaming.config import Config
+from dnaStreaming.services import pubsub_service, credentials_service
+
+
+class Listener(object):
+    DEFAULT_UNLIMITED_MESSAGES = None
+
+    def __init__(self, service_account_id=None, user_key=None, config_file=None):
+        config = Config(service_account_id, user_key, config_file)
+        self._initialize(config)
+        self.current_subscription_index = 0
+
+    def _initialize(self, config):
+        self.config = config
+
+    def listen(self, on_message_callback, maximum_messages=DEFAULT_UNLIMITED_MESSAGES, subscription_id="", batch_size=10):
+        pubsub_client = pubsub_service.get_client(self.config)
+
+        subscription_id = subscription_id or self.config.subscription()
+        if not subscription_id:
+            raise Exception(
+                'No subscription specified. You must specify the subscription ID either through an environment ' +
+                'variable, a config file or by passing the value to the method.')
+
+        streaming_credentials = credentials_service.fetch_credentials(
+            self.config)
+        subscription_path = pubsub_client.subscription_path(
+            streaming_credentials['project_id'], subscription_id)
+
+        logger.info(
+            'Listeners for subscriptions have been configured, set and await message arrival.')
+
+        count = 0
+        while maximum_messages is None or count < maximum_messages:
+            try:
+                if maximum_messages is not None:
+                    batch_size = min(batch_size, maximum_messages - count)
+
+                pull_request = pubsub_v1.PullRequest(
+                    subscription=subscription_path,
+                    max_messages=batch_size,
+                )
+                results = pubsub_client.pull(request=pull_request)
+
+                if results:
+                    if len(results.received_messages) > 0:
+                        for message in results.received_messages:
+
+                            pubsub_msg = json.loads(message.message.data)
+                            logger.info("Received news message with ID: {}".format(
+                                pubsub_msg['data'][0]['id']))
+                            news_msg = pubsub_msg['data'][0]['attributes']
+
+                            keep_message_flow = on_message_callback(
+                                news_msg, subscription_id)
+
+                            if not keep_message_flow:
+                                logger.info(
+                                    'Callback function returned False or falsy. Message flow stopped.')
+                                return
+
+                            ack_request = {
+                                "subscription": subscription_path,
+                                "ack_ids": [message.ack_id],
+                            }
+                            pubsub_client.acknowledge(request=ack_request)
+                            count += 1
+
+            except GoogleAPICallError as e:
+                if isinstance(e, NotFound):
+                    raise e
+                logger.error(
+                    "Encountered a problem while trying to pull a message from a stream. Error is as follows: {}".format(str(e)))
+                logger.error("Due to the previous error, system will pause 10 seconds. System will then attempt to pull the message from "
+                             "the stream again.")
+                time.sleep(10)
+                pubsub_client = pubsub_service.get_client(self.config)
+
+    def listen_async(self, on_message_callback, subscription_id=""):
+        def ack_message_and_callback(message):
+            pubsub_msg = json.loads(message.data)
+            logger.info("Received news message with ID: {}".format(
+                pubsub_msg['data'][0]['id']))
+            news_msg = pubsub_msg['data'][0]['attributes']
+            on_message_callback(news_msg, subscription_id)
+            message.ack()
+
+        pubsub_client = pubsub_service.get_client(self.config)
+
+        subscription_id = subscription_id or self.config.subscription()
+
+        if not subscription_id:
+            raise Exception(
+                'No subscription specified. You must specify the subscription ID either through an environment variable, a config file or '
+                'by passing the value to the method.')
+
+        streaming_credentials = credentials_service.fetch_credentials(
+            self.config)
+        subscription_path = pubsub_client.subscription_path(
+            streaming_credentials['project_id'], subscription_id)
+        subscription = pubsub_client.subscribe(
+            subscription_path, callback=ack_message_and_callback)
+        logger.info(
+            'Listeners for subscriptions have been configured, set and await message arrival.')
+        return subscription

{dnaStreaming-2.0.2 → dnastreaming-2.1.0}/dnaStreaming/services/credentials_service.py

@@ -5,7 +5,7 @@ import requests
 
 
 def fetch_credentials(config):
-    response = _get_requests().get(config.get_uri_context() + '/accounts/streaming-credentials',
+    response = _get_requests().get(config.get_uri_context() + '/sns-accounts/streaming-credentials',
                                    headers=config.get_headers())
 
     if response.status_code == 401: