dnaStreaming 2.3.0__tar.gz → 2.3.1__tar.gz

{dnastreaming-2.3.0/dnaStreaming.egg-info → dnastreaming-2.3.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.3.0
+Version: 2.3.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.0.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT
@@ -220,7 +220,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -258,14 +258,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

{dnastreaming-2.3.0 → dnastreaming-2.3.1}/README.rst

@@ -194,7 +194,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -232,14 +232,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

dnastreaming-2.3.1/demo/pubsub_stream.py

@@ -0,0 +1,81 @@
+import os
+import json
+import datetime
+from google.cloud import pubsub_v1
+from dnaStreaming.listener import Listener
+from dnaStreaming import logger
+
+gcp_project_id = os.getenv('GCP_PROJECT_ID', None)
+gcp_pubsub_topic = os.getenv('GCP_PUBSUB_TOPIC', None)
+gcp_creds = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+if gcp_project_id is not None and gcp_pubsub_topic is not None and gcp_creds is not None:
+    listener = Listener()
+    ps_publisher = pubsub_v1.PublisherClient()
+    topic_path = ps_publisher.topic_path(gcp_project_id, gcp_pubsub_topic)
+    print(f"\n[ACTIVITY] Sending messages to Pub/Sub topic {gcp_pubsub_topic} in GCP...")
+
+    def callback(factiva_message, subscription_id, file_handle=None):
+        try:
+            if 'action' in factiva_message.keys():
+                # Message is an article event
+                # Implement your logic according to the documentation:
+                # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+                if factiva_message['action'] == 'add':
+                    # Insert the article to the repository as new. Handle repeated messages.
+                    print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+                elif factiva_message['action'] == 'rep':
+                    # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                    print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+                elif factiva_message['action'] == 'del':
+                    # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                    print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+                else:
+                    print_message(f"Factiva Action Not Handled: {factiva_message['action']}")
+                
+                logger.info(f"Sent to GCP Pub/Sub, message with AN: {factiva_message.get('an', 'N/A')}")
+                
+            elif 'event_type' in factiva_message.keys():
+                # Message is a bulk action or service event
+                if factiva_message['event_type'] == 'source_delete':
+                    # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                    print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+                else:
+                    print_message(f"Factiva Event Type Not Handled: {factiva_message['event_type']}")
+
+                logger.info(f"Sent to GCP Pub/Sub, message with event: {factiva_message.get('event_type', 'N/A')}")
+                
+            else:
+                print_message(f"Unexpected Message Format:[{factiva_message}]")
+                
+            # Publish the message to GCP Pub/Sub
+            m_data = json.dumps(factiva_message, ensure_ascii=False).encode("utf-8")
+            ps_publisher.publish(topic_path, data=m_data)
+            
+            callback.counter += 1
+        
+            if callback.counter % 100 == 0:
+                print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+                
+            return True
+        
+        except Exception as e:
+            logger.error(f"Error processing Factiva message: {e}")
+            # Only return False if you want to stop the listener
+            return True
+
+    callback.counter = 0
+    print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+    listener.listen(callback)
+    
+else:
+    print("[ERROR]: Required ENV variables not set")
+    if gcp_project_id is None:
+        print("    - GCP_PROJECT_ID: GCP Project ID")
+    if gcp_pubsub_topic is None:
+        print("    - GCP_PUBSUB_TOPIC: GCP Pub/Sub Topic Name")
+    if gcp_creds is None:
+        print("    - GOOGLE_APPLICATION_CREDENTIALS: Path to Service Account JSON Credentials File")

dnastreaming-2.3.1/demo/show_ha_stream_async.py

@@ -0,0 +1,62 @@
+import os
+import datetime
+from time import sleep
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+max_secs = 10
+print(f"\n[ACTIVITY] Receiving messages (ASYNC) for {max_secs} seconds...")
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+def callback(factiva_message, subscription_id):
+    try:
+        if 'action' in factiva_message.keys():
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+            
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+                
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+            
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+                
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+    return True
+
+callback.counter = 0
+listener_controller = listener.listen_async_ha(callback)
+
+# Stop receiving messages after max_secs
+for count in range(0, max_secs):
+    sleep(1)
+
+if listener_controller.listener_is_running():
+    listener_controller.stop_listener()
+
+print("stop receiving messages")

dnastreaming-2.3.1/demo/show_stream.py

@@ -0,0 +1,57 @@
+import os
+import datetime
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+
+print("\n[ACTIVITY] Receiving messages (SYNC)...")
+
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+
+def callback(factiva_message: dict, subscription_id: str) -> bool:
+    try:
+        if 'action' in factiva_message.keys(): 
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+            
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+    
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+
+    return True
+
+
+callback.counter = 0
+print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+listener.listen(callback)

dnastreaming-2.3.1/demo/show_stream_async.py

@@ -0,0 +1,61 @@
+import os
+import datetime
+from time import sleep
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+max_secs = 10
+print(f"\n[ACTIVITY] Receiving messages (ASYNC) for {max_secs} seconds...")
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+def callback(factiva_message, subscription_id):
+    try:
+        if 'action' in factiva_message.keys():
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+            
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+    return True
+
+callback.counter = 0
+print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+future = listener.listen_async(callback)
+
+# Stop receiving messages after 5 seconds
+for count in range(0, max_secs):
+    sleep(1)
+
+if future.running():
+    future.cancel()
+
+print("stop receiving messages")

{dnastreaming-2.3.0 → dnastreaming-2.3.1/dnaStreaming.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.3.0
+Version: 2.3.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.0.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT
@@ -220,7 +220,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -258,14 +258,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

{dnastreaming-2.3.0 → dnastreaming-2.3.1}/dnaStreaming.egg-info/SOURCES.txt

@@ -3,6 +3,11 @@ MANIFEST.in
 README.rst
 setup.cfg
 setup.py
+demo/__init__.py
+demo/pubsub_stream.py
+demo/show_ha_stream_async.py
+demo/show_stream.py
+demo/show_stream_async.py
 dnaStreaming/__init__.py
 dnaStreaming/config.py
 dnaStreaming/customer_config.json
@@ -12,11 +17,6 @@ dnaStreaming.egg-info/SOURCES.txt
 dnaStreaming.egg-info/dependency_links.txt
 dnaStreaming.egg-info/requires.txt
 dnaStreaming.egg-info/top_level.txt
-dnaStreaming/demo/__init__.py
-dnaStreaming/demo/pubsub_stream.py
-dnaStreaming/demo/show_ha_stream_async.py
-dnaStreaming/demo/show_stream.py
-dnaStreaming/demo/show_stream_async.py
 dnaStreaming/services/__init__.py
 dnaStreaming/services/authentication_service.py
 dnaStreaming/services/availability_service.py

{dnastreaming-2.3.0 → dnastreaming-2.3.1}/dnaStreaming.egg-info/top_level.txt

@@ -1 +1,2 @@
+demo
 dnaStreaming

{dnastreaming-2.3.0 → dnastreaming-2.3.1}/setup.py

@@ -6,7 +6,7 @@ this_directory = path.abspath(path.dirname(__file__))
 with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
-VERSION = "2.3.0"
+VERSION = "2.3.1"
 RELEASE_TAG = f"release-{VERSION}"
 
 setup(

dnastreaming-2.3.0/dnaStreaming/demo/pubsub_stream.py

@@ -1,38 +0,0 @@
-import os
-import json
-from google.cloud import pubsub_v1
-from dnaStreaming.listener import Listener
-from dnaStreaming import logger
-
-gcp_project_id = os.getenv('GCP_PROJECT_ID', None)
-gcp_pubsub_topic = os.getenv('GCP_PUBSUB_TOPIC', None)
-gcp_creds = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
-
-if gcp_project_id is not None and gcp_pubsub_topic is not None and gcp_creds is not None:
-    listener = Listener()
-    ps_publisher = pubsub_v1.PublisherClient()
-    topic_path = ps_publisher.topic_path(gcp_project_id, gcp_pubsub_topic)
-    print("\n[ACTIVITY] Sending messages to Pub/Sub topic {} in GCP\n[0]".format(gcp_pubsub_topic), end='')
-
-    def callback(message, subscription_id, file_handle=None):
-        callback.counter += 1
-        logger.info("Processing article with ID: {}".format(message['an']))
-        m_data = json.dumps(message, ensure_ascii=False).encode("utf-8")
-        ps_publisher.publish(topic_path, data=m_data)
-        logger.info("Sent to GCP Pub/Sub, article with ID: {}".format(message['an']))
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-        return True
-
-    callback.counter = 0
-    listener.listen(callback)
-else:
-    print("[ERROR]: Required ENV variables not set")
-    if gcp_project_id is None:
-        print("    - GCP_PROJECT_ID: GCP Project ID")
-    if gcp_pubsub_topic is None:
-        print("    - GCP_PUBSUB_TOPIC: GCP Pub/Sub Topic Name")
-    if gcp_creds is None:
-        print("    - GOOGLE_APPLICATION_CREDENTIALS: Path to Service Account JSON Credentials File")

dnastreaming-2.3.0/dnaStreaming/demo/show_ha_stream_async.py

@@ -1,36 +0,0 @@
-import os
-from time import sleep
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-max_secs = 1000
-print("\n[ACTIVITY] Receiving messages (ASYNC) for {} seconds...\n[0]".format(max_secs), end='')
-
-
-def callback(message, subscription_id):
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, TITLE: {}'.format(subscription_id, callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, *** DELETE ***'.format(subscription_id, callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-listener_controller = listener.listen_async_ha(callback)
-
-# Stop receiving messages after 5 seconds
-for count in range(0, max_secs):
-    sleep(1)
-
-if listener_controller.listener_is_running():
-    listener_controller.stop_listener()
-
-print("stop receiving messages")

dnastreaming-2.3.0/dnaStreaming/demo/show_stream.py

@@ -1,30 +0,0 @@
-import os
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-print("\n[ACTIVITY] Receiving messages (SYNC)...\n[0]", end='')
-
-
-def callback(message, subscription_id):
-    # Replace the code below with the custom operations you'd like to apply to each article (message)
-    # like writing to a file, database, or pushing the message to a separate message broker system.
-    # This demo only prints some message details.
-    # Subscription id parameter is provided and can be used for auditing purposes.
-
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [MSG] [{}]: AN: {}, TITLE: {}'.format(callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [MSG] [{}]: AN: {}, *** DELETE ***'.format(callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-listener.listen(callback)

dnastreaming-2.3.0/dnaStreaming/demo/show_stream_async.py

@@ -1,36 +0,0 @@
-import os
-from time import sleep
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-max_secs = 5
-print("\n[ACTIVITY] Receiving messages (ASYNC) for {} seconds...\n[0]".format(max_secs), end='')
-
-
-def callback(message, subscription_id):
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [MSG] [{}]: AN: {}, TITLE: {}'.format(callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [MSG] [{}]: AN: {}, *** DELETE ***'.format(callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-future = listener.listen_async(callback)
-
-# Stop receiving messages after 5 seconds
-for count in range(0, max_secs):
-    sleep(1)
-
-if future.running():
-    future.cancel()
-
-print("stop receiving messages")