dnaStreaming 2.1.0__tar.gz → 2.2.1__tar.gz

{dnastreaming-2.1.0/dnaStreaming.egg-info → dnastreaming-2.2.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.1.0
+Version: 2.2.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.1.0.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.2.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT

dnastreaming-2.2.1/dnaStreaming/demo/show_ha_stream_async.py

@@ -0,0 +1,36 @@
+import os
+from time import sleep
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
+max_secs = 1000
+print("\n[ACTIVITY] Receiving messages (ASYNC) for {} seconds...\n[0]".format(max_secs), end='')
+
+
+def callback(message, subscription_id):
+    callback.counter += 1
+    if not quiet_demo:
+        if message['action'] != 'del':
+            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, TITLE: {}'.format(subscription_id, callback.counter, message['an'], message['title']))
+        else:
+            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, *** DELETE ***'.format(subscription_id, callback.counter, message['an']))
+    else:
+        if callback.counter % 10 == 0:
+            print('[{}]'.format(callback.counter), end='')
+        else:
+            print('.', end='')
+    return True
+
+
+callback.counter = 0
+listener_controller = listener.listen_async_ha(callback)
+
+# Stop receiving messages after 5 seconds
+for count in range(0, max_secs):
+    sleep(1)
+
+if listener_controller.listener_is_running():
+    listener_controller.stop_listener()
+
+print("stop receiving messages")

{dnastreaming-2.1.0 → dnastreaming-2.2.1}/dnaStreaming/listener.py

@@ -1,7 +1,7 @@
 import time
 import requests
 import json
-from threading import Thread
+from threading import Thread, Event
 
 from google.api_core.exceptions import GoogleAPICallError, NotFound
 from google import pubsub_v1
@@ -9,7 +9,19 @@ from google import pubsub_v1
 from dnaStreaming import logger
 from dnaStreaming.config import Config
 from dnaStreaming.services import pubsub_service, credentials_service
+from dnaStreaming.services.availability_service import MAIN_REGION, BACKUP_REGION, ha_listen
 
+class ListenerController(object):
+    def __init__(self, thread, stop_event):
+        self.thread = thread
+        self.stop_event = stop_event
+
+    def stop_listener(self):
+        self.stop_event.set()
+        self.thread.join()
+
+    def listener_is_running(self):
+        self.thread.is_alive()
 
 class Listener(object):
     DEFAULT_UNLIMITED_MESSAGES = None
@@ -112,3 +124,58 @@ class Listener(object):
         logger.info(
             'Listeners for subscriptions have been configured, set and await message arrival.')
         return subscription
+
+    def listen_async_ha(self, on_message_callback):
+        def ack_message_and_callback(message, subscription_path):
+            pubsub_msg = json.loads(message.data)
+            logger.info("Received news message with ID: {}".format(
+                pubsub_msg['data'][0]['id']))
+            news_msg = pubsub_msg['data'][0]['attributes']
+            short_subscription_id = subscription_path.split("/")[-1]
+            on_message_callback(news_msg, short_subscription_id)
+            message.ack()
+
+        main_pubsub_client = pubsub_service.get_client(self.config, MAIN_REGION)
+        backup_pubsub_client = pubsub_service.get_client(self.config, BACKUP_REGION)
+
+        subscription_id = self.config.subscription()
+
+        if not subscription_id:
+            raise Exception(
+                'No subscription specified. You must specify the subscription ID either through an environment variable, a config file or '
+                'by passing the value to the method.')
+
+        streaming_credentials = credentials_service.fetch_credentials(
+            self.config)
+
+        api_host = self.config.get_uri_context()
+        user_key = self.config.get_user_key()
+
+        main_subscription_id = subscription_id if "-bak-" not in subscription_id else subscription_id.replace("-filtered-bak-", "-filtered-")
+        backup_subscription_id = subscription_id if "-bak-" in subscription_id else subscription_id.replace("-filtered-", "-filtered-bak-")
+
+        main_subscription_path = main_pubsub_client.subscription_path(
+            streaming_credentials['project_id'], main_subscription_id)
+        backup_subscription_path = backup_pubsub_client.subscription_path(
+            streaming_credentials['project_id'], backup_subscription_id)
+
+        stop_event = Event()
+        listener_thread = Thread(target=ha_listen, daemon=True, args=(
+            api_host,
+            user_key,
+            subscription_id,
+            stop_event,
+            main_subscription_path,
+            backup_subscription_path,
+            main_pubsub_client,
+            backup_pubsub_client,
+            ack_message_and_callback
+        ))
+        listener_thread.start()
+
+        listener_controller = ListenerController(listener_thread, stop_event)
+
+        logger.info(
+            'Listeners for subscriptions have been configured, set and await message arrival.')
+
+        return listener_controller

dnastreaming-2.2.1/dnaStreaming/services/availability_service.py

@@ -0,0 +1,87 @@
+import requests
+import time
+
+from dnaStreaming import logger
+
+MAIN_REGION = "us-central1"
+BACKUP_REGION = "us-east5"
+COOLDOWN_PERIOD = 5 * 60
+
+def get_active_region(api_host, subscription_id, user_key):
+
+    try:
+        ha_status_url = f"{api_host}/stream_health/get_active_region"
+
+        headers = {
+            "subscription-id": subscription_id,
+            "user-key": user_key
+        }
+
+        response = requests.get(url=ha_status_url, headers=headers)
+
+        assert response.status_code == 200, "Failed to fetch active region"
+
+        payload = response.json()
+
+        active_region = payload["data"]["active_region"]
+
+        return active_region
+
+    except:
+
+        return None
+
+
+def ha_listen(api_host, user_key, subscription_id, stop_event, main_subscription_path, backup_subscription_path, main_pubsub_client, backup_pubsub_client, callback):
+
+    # The listener starts reading messages from the main region by default.
+
+    current_region = MAIN_REGION
+
+    def wrapped_callback(subscription_path):
+        def inner_callback(message):
+            callback(message, subscription_path)
+
+        return inner_callback
+
+    streaming_pull_future = main_pubsub_client.subscribe(
+        main_subscription_path, callback=wrapped_callback(main_subscription_path))
+    
+    while not stop_event.is_set():
+
+        active_region = get_active_region(api_host, subscription_id, user_key)
+
+        if active_region is None or active_region not in (MAIN_REGION, BACKUP_REGION):
+            logger.warning(f"Got invalid region from API: {active_region}. Listener will keep reading from region {current_region}")
+            continue
+        
+        if current_region != active_region:
+
+            logger.warning(f"Switch event detected in, switching from region {current_region} to region {active_region}")
+            
+            logger.warning(f"Stopping listener in region {current_region}...")
+
+            # We stop the previous listening process
+            streaming_pull_future.cancel()
+            streaming_pull_future.result()
+
+            logger.warning(f"Stopped listener in region {current_region}")
+
+            logger.warning(f"Starting listener in region {active_region}...")
+
+            if active_region == MAIN_REGION:
+                streaming_pull_future = main_pubsub_client.subscribe(
+                    main_subscription_path, callback=wrapped_callback(main_subscription_path))
+            else: # active_region == BACKUP_REGION
+                streaming_pull_future = backup_pubsub_client.subscribe(
+                    backup_subscription_path, callback=wrapped_callback(backup_subscription_path))
+                
+            logger.warning(f"Started listener in region {active_region}")
+
+            current_region = active_region
+        
+        # We wait 5 minutes for cooldown for next call to API
+        time.sleep(COOLDOWN_PERIOD)
+
+    streaming_pull_future.cancel()
+    streaming_pull_future.result()

dnastreaming-2.2.1/dnaStreaming/services/pubsub_service.py

@@ -0,0 +1,19 @@
+from __future__ import absolute_import, division, print_function
+
+from google.cloud.pubsub_v1 import SubscriberClient
+
+from dnaStreaming.services import authentication_service
+from dnaStreaming.services import credentials_service
+from dnaStreaming.services.availability_service import MAIN_REGION, BACKUP_REGION
+
+
+def get_client(config, region=None):
+    streaming_credentials = credentials_service.fetch_credentials(config)
+    credentials = authentication_service.get_authenticated_oauth_credentials(streaming_credentials)
+
+    if region in (MAIN_REGION, BACKUP_REGION):
+        client_options = {"api_endpoint": f"{region}-pubsub.googleapis.com:443"}
+    else:
+        client_options = {"api_endpoint": "pubsub.googleapis.com:443"}
+
+    return SubscriberClient(credentials=credentials, client_options=client_options)

{dnastreaming-2.1.0 → dnastreaming-2.2.1/dnaStreaming.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.1.0
+Version: 2.2.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.1.0.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.2.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT

{dnastreaming-2.1.0 → dnastreaming-2.2.1}/dnaStreaming.egg-info/SOURCES.txt

@@ -14,10 +14,12 @@ dnaStreaming.egg-info/requires.txt
 dnaStreaming.egg-info/top_level.txt
 dnaStreaming/demo/__init__.py
 dnaStreaming/demo/pubsub_stream.py
+dnaStreaming/demo/show_ha_stream_async.py
 dnaStreaming/demo/show_stream.py
 dnaStreaming/demo/show_stream_async.py
 dnaStreaming/services/__init__.py
 dnaStreaming/services/authentication_service.py
+dnaStreaming/services/availability_service.py
 dnaStreaming/services/credentials_service.py
 dnaStreaming/services/pubsub_service.py
 dnaStreaming/test/__init__.py

{dnastreaming-2.1.0 → dnastreaming-2.2.1}/setup.py

@@ -6,7 +6,7 @@ this_directory = path.abspath(path.dirname(__file__))
 with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
-VERSION = "2.1.0"
+VERSION = "2.2.1"
 RELEASE_TAG = f"release-{VERSION}"
 
 setup(

dnastreaming-2.1.0/dnaStreaming/services/pubsub_service.py

@@ -1,13 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-from google.cloud.pubsub_v1 import SubscriberClient
-
-from dnaStreaming.services import authentication_service
-from dnaStreaming.services import credentials_service
-
-
-def get_client(config):
-    streaming_credentials = credentials_service.fetch_credentials(config)
-    credentials = authentication_service.get_authenticated_oauth_credentials(streaming_credentials)
-
-    return SubscriberClient(credentials=credentials)