dnaStreaming 2.2.1.dev1__tar.gz → 2.3.1__tar.gz

{dnastreaming-2.2.1.dev1/dnaStreaming.egg-info → dnastreaming-2.3.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.2.1.dev1
+Version: 2.3.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.2.1.dev1.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT
@@ -14,6 +14,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/x-rst
 License-File: LICENSE
+Requires-Dist: setuptools>=75.1.0
 Requires-Dist: googleapis-common-protos>=1.56.4
 Requires-Dist: google-auth>=2.11.0
 Requires-Dist: google-cloud-pubsub==2.13.6
@@ -66,7 +67,13 @@ To set your subscription ID, simply set an environment variable named 'SUBSCRIPT
     export SUBSCRIPTION_ID="ABC1234567889"
 
 
-The code above is the command line expression for setting this environment variable on Mac OSX. Other operating systems might have a slightly different techniques for setting environment variables on the command line.
+To set your log folder path, simply set this folder variable named 'LOG_PATH' like so
+
+.. code-block::
+
+    export LOG_PATH="/your/custom/log/path"
+
+
 
 2. Using the configuration file.
 ###################################################################
@@ -146,7 +153,25 @@ You may want to listen messages asynchronously like so:
 Log Files
 _________
 
-Very minimal logging is written to the module's path 'logs/dj-dna-streaming-python.log'. To keep maintenance simple this log is overwritten every time the app starts.
+
+Minimal logging is written to a file named `dj-dna-streaming-python.log`.
+
+By default, logs are written to the first available directory from the following list:
+
+1. A custom path set via the environment variable `LOG_PATH`.
+2. A `logs/` folder located within the package installation directory.
+3. A fallback directory: `~/.dj-dna-streaming-python/logs/`.
+
+The first writable location found is selected. A message like `Will log to: /your/custom/log/path` is printed to the console on startup.
+
+💡 The log file is overwritten each time the application starts to keep maintenance simple.
+
+You can specify:
+
+- **Absolute paths**: For example, `/var/log/dna-streaming`.
+- **Relative paths**: For example, `./logs`, relative to the current working directory at runtime.
+
+The code verifies that the specified path is writable. If it isn’t, it automatically falls back to the next available option.
 
 
 Testing
@@ -195,7 +220,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -233,14 +258,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1}/README.rst

@@ -41,7 +41,13 @@ To set your subscription ID, simply set an environment variable named 'SUBSCRIPT
     export SUBSCRIPTION_ID="ABC1234567889"
 
 
-The code above is the command line expression for setting this environment variable on Mac OSX. Other operating systems might have a slightly different techniques for setting environment variables on the command line.
+To set your log folder path, simply set this folder variable named 'LOG_PATH' like so
+
+.. code-block::
+
+    export LOG_PATH="/your/custom/log/path"
+
+
 
 2. Using the configuration file.
 ###################################################################
@@ -121,7 +127,25 @@ You may want to listen messages asynchronously like so:
 Log Files
 _________
 
-Very minimal logging is written to the module's path 'logs/dj-dna-streaming-python.log'. To keep maintenance simple this log is overwritten every time the app starts.
+
+Minimal logging is written to a file named `dj-dna-streaming-python.log`.
+
+By default, logs are written to the first available directory from the following list:
+
+1. A custom path set via the environment variable `LOG_PATH`.
+2. A `logs/` folder located within the package installation directory.
+3. A fallback directory: `~/.dj-dna-streaming-python/logs/`.
+
+The first writable location found is selected. A message like `Will log to: /your/custom/log/path` is printed to the console on startup.
+
+💡 The log file is overwritten each time the application starts to keep maintenance simple.
+
+You can specify:
+
+- **Absolute paths**: For example, `/var/log/dna-streaming`.
+- **Relative paths**: For example, `./logs`, relative to the current working directory at runtime.
+
+The code verifies that the specified path is writable. If it isn’t, it automatically falls back to the next available option.
 
 
 Testing
@@ -170,7 +194,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -208,14 +232,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

dnastreaming-2.3.1/demo/pubsub_stream.py

@@ -0,0 +1,81 @@
+import os
+import json
+import datetime
+from google.cloud import pubsub_v1
+from dnaStreaming.listener import Listener
+from dnaStreaming import logger
+
+gcp_project_id = os.getenv('GCP_PROJECT_ID', None)
+gcp_pubsub_topic = os.getenv('GCP_PUBSUB_TOPIC', None)
+gcp_creds = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+if gcp_project_id is not None and gcp_pubsub_topic is not None and gcp_creds is not None:
+    listener = Listener()
+    ps_publisher = pubsub_v1.PublisherClient()
+    topic_path = ps_publisher.topic_path(gcp_project_id, gcp_pubsub_topic)
+    print(f"\n[ACTIVITY] Sending messages to Pub/Sub topic {gcp_pubsub_topic} in GCP...")
+
+    def callback(factiva_message, subscription_id, file_handle=None):
+        try:
+            if 'action' in factiva_message.keys():
+                # Message is an article event
+                # Implement your logic according to the documentation:
+                # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+                if factiva_message['action'] == 'add':
+                    # Insert the article to the repository as new. Handle repeated messages.
+                    print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+                elif factiva_message['action'] == 'rep':
+                    # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                    print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+                elif factiva_message['action'] == 'del':
+                    # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                    print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+                else:
+                    print_message(f"Factiva Action Not Handled: {factiva_message['action']}")
+                
+                logger.info(f"Sent to GCP Pub/Sub, message with AN: {factiva_message.get('an', 'N/A')}")
+                
+            elif 'event_type' in factiva_message.keys():
+                # Message is a bulk action or service event
+                if factiva_message['event_type'] == 'source_delete':
+                    # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                    print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+                else:
+                    print_message(f"Factiva Event Type Not Handled: {factiva_message['event_type']}")
+
+                logger.info(f"Sent to GCP Pub/Sub, message with event: {factiva_message.get('event_type', 'N/A')}")
+                
+            else:
+                print_message(f"Unexpected Message Format:[{factiva_message}]")
+                
+            # Publish the message to GCP Pub/Sub
+            m_data = json.dumps(factiva_message, ensure_ascii=False).encode("utf-8")
+            ps_publisher.publish(topic_path, data=m_data)
+            
+            callback.counter += 1
+        
+            if callback.counter % 100 == 0:
+                print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+                
+            return True
+        
+        except Exception as e:
+            logger.error(f"Error processing Factiva message: {e}")
+            # Only return False if you want to stop the listener
+            return True
+
+    callback.counter = 0
+    print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+    listener.listen(callback)
+    
+else:
+    print("[ERROR]: Required ENV variables not set")
+    if gcp_project_id is None:
+        print("    - GCP_PROJECT_ID: GCP Project ID")
+    if gcp_pubsub_topic is None:
+        print("    - GCP_PUBSUB_TOPIC: GCP Pub/Sub Topic Name")
+    if gcp_creds is None:
+        print("    - GOOGLE_APPLICATION_CREDENTIALS: Path to Service Account JSON Credentials File")

dnastreaming-2.3.1/demo/show_ha_stream_async.py

@@ -0,0 +1,62 @@
+import os
+import datetime
+from time import sleep
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+max_secs = 10
+print(f"\n[ACTIVITY] Receiving messages (ASYNC) for {max_secs} seconds...")
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+def callback(factiva_message, subscription_id):
+    try:
+        if 'action' in factiva_message.keys():
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+            
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+                
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+            
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+                
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+    return True
+
+callback.counter = 0
+listener_controller = listener.listen_async_ha(callback)
+
+# Stop receiving messages after max_secs
+for count in range(0, max_secs):
+    sleep(1)
+
+if listener_controller.listener_is_running():
+    listener_controller.stop_listener()
+
+print("stop receiving messages")

dnastreaming-2.3.1/demo/show_stream.py

@@ -0,0 +1,57 @@
+import os
+import datetime
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+
+print("\n[ACTIVITY] Receiving messages (SYNC)...")
+
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+
+def callback(factiva_message: dict, subscription_id: str) -> bool:
+    try:
+        if 'action' in factiva_message.keys(): 
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+            
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+    
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+
+    return True
+
+
+callback.counter = 0
+print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+listener.listen(callback)

dnastreaming-2.3.1/demo/show_stream_async.py

@@ -0,0 +1,61 @@
+import os
+import datetime
+from time import sleep
+from dnaStreaming.listener import Listener
+
+listener = Listener()
+max_secs = 10
+print(f"\n[ACTIVITY] Receiving messages (ASYNC) for {max_secs} seconds...")
+
+def print_message(message):
+    print(f"[{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {message}")
+
+def callback(factiva_message, subscription_id):
+    try:
+        if 'action' in factiva_message.keys():
+            # Message is an article event
+            # Implement your logic according to the documentation:
+            # https://developer.dowjones.com/documents/site-docs-factiva_apis-factiva_analytics_apis-factiva_streams_api#article-specific-events
+            if factiva_message['action'] == 'add':
+                # Insert the article to the repository as new. Handle repeated messages.
+                print_message(f"[ARTICLE] [ADD] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'rep':
+                # Upsert/Update/AddNewVersion the article in the repository. Handle repeated messages.
+                print_message(f"[ARTICLE] [REP] AN: {factiva_message['an']} - {factiva_message['title']}")
+            elif factiva_message['action'] == 'del':
+                # Delete or mark as deleted the article in the repository. Handle inexistent article AN.
+                print_message(f"[ARTICLE] [DEL] AN: {factiva_message['an']} - *** DELETE ***")
+            else:
+                print_message(f"[ERROR] Factiva Action Not Handled: {factiva_message['action']}")
+        elif 'event_type' in factiva_message.keys():
+            # Message is a bulk action or service event
+            if factiva_message['event_type'] == 'source_delete':
+                # Delete all articles from the repository matching the source_code. Handle inexistent source_code and repeated messages.
+                print_message(f"[EVENT] [SOURCE_DELETE] Source: {factiva_message['source_code'].upper()} - {factiva_message['description']}")
+            else:
+                print_message(f"[ERROR] Factiva Event Type Not Handled: {factiva_message['event_type']}")
+        else:
+            print_message(f"[ERROR] Unexpected Message Format:[{factiva_message}]")
+
+        callback.counter += 1
+        if callback.counter % 100 == 0:
+            print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+            
+    except Exception as e:
+        print_message(f"[ERROR] Error processing Factiva message: {e}")
+        # Only return False if you want to stop the listener
+        return True
+    return True
+
+callback.counter = 0
+print_message(f"[INFO] *** Processed {callback.counter} messages ***")
+future = listener.listen_async(callback)
+
+# Stop receiving messages after 5 seconds
+for count in range(0, max_secs):
+    sleep(1)
+
+if future.running():
+    future.cancel()
+
+print("stop receiving messages")

dnastreaming-2.3.1/dnaStreaming/__init__.py

@@ -0,0 +1,45 @@
+from __future__ import absolute_import, division, print_function
+
+import os
+import sys
+import logging
+
+
+def get_log_path():
+    env_log_path = os.getenv("LOG_PATH")
+    fallback_log_dir = os.path.expanduser('~/.dj-dna-streaming-python/logs')
+    base_dir = os.path.dirname(__file__)
+    default_path = os.path.join(base_dir, 'logs')
+
+    candidates = [env_log_path, default_path, fallback_log_dir]
+
+    for path in candidates:
+        if path:
+            try:
+                os.makedirs(path, exist_ok=True)
+                testfile = os.path.join(path, '.write_test')
+                with open(testfile, 'w') as f:
+                    f.write('test')
+                os.remove(testfile)
+                return path
+            except Exception as e:
+                print(f"WARNING: Cannot write to '{path}': {e}")
+
+    raise RuntimeError("ERROR: Could not find a writable log directory.")
+
+
+log_path = get_log_path()
+print("Will log to: {}".format(log_path))
+
+# Logging setup
+logging.basicConfig(level=logging.WARN)
+logger = logging.getLogger()
+
+fileHandler = logging.FileHandler(os.path.join(log_path, 'dj-dna-streaming-python.log'))
+logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
+fileHandler.setFormatter(logFormatter)
+logger.addHandler(fileHandler)
+
+consoleHandler = logging.StreamHandler(sys.stdout)
+consoleHandler.setFormatter(logFormatter)
+logger.addHandler(consoleHandler)

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1/dnaStreaming.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: dnaStreaming
-Version: 2.2.1.dev1
+Version: 2.3.1
 Summary: Dow Jones DNA Streaming Project
 Home-page: https://github.com/dowjones/dj-dna-streams-python
-Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.2.1.dev1.tar.gz
+Download-URL: https://github.com/dowjones/dj-dna-streams-python/archive/release-2.3.1.tar.gz
 Author: Zachary Kagan
 Author-email: zachary.kagan@dowjones.com
 License: MIT
@@ -14,6 +14,7 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Description-Content-Type: text/x-rst
 License-File: LICENSE
+Requires-Dist: setuptools>=75.1.0
 Requires-Dist: googleapis-common-protos>=1.56.4
 Requires-Dist: google-auth>=2.11.0
 Requires-Dist: google-cloud-pubsub==2.13.6
@@ -66,7 +67,13 @@ To set your subscription ID, simply set an environment variable named 'SUBSCRIPT
     export SUBSCRIPTION_ID="ABC1234567889"
 
 
-The code above is the command line expression for setting this environment variable on Mac OSX. Other operating systems might have a slightly different techniques for setting environment variables on the command line.
+To set your log folder path, simply set this folder variable named 'LOG_PATH' like so
+
+.. code-block::
+
+    export LOG_PATH="/your/custom/log/path"
+
+
 
 2. Using the configuration file.
 ###################################################################
@@ -146,7 +153,25 @@ You may want to listen messages asynchronously like so:
 Log Files
 _________
 
-Very minimal logging is written to the module's path 'logs/dj-dna-streaming-python.log'. To keep maintenance simple this log is overwritten every time the app starts.
+
+Minimal logging is written to a file named `dj-dna-streaming-python.log`.
+
+By default, logs are written to the first available directory from the following list:
+
+1. A custom path set via the environment variable `LOG_PATH`.
+2. A `logs/` folder located within the package installation directory.
+3. A fallback directory: `~/.dj-dna-streaming-python/logs/`.
+
+The first writable location found is selected. A message like `Will log to: /your/custom/log/path` is printed to the console on startup.
+
+💡 The log file is overwritten each time the application starts to keep maintenance simple.
+
+You can specify:
+
+- **Absolute paths**: For example, `/var/log/dna-streaming`.
+- **Relative paths**: For example, `./logs`, relative to the current working directory at runtime.
+
+The code verifies that the specified path is writable. If it isn’t, it automatically falls back to the next available option.
 
 
 Testing
@@ -195,7 +220,7 @@ If you are enhancing this codebase (and not just using it as a library), follow
     virtualenv venv
 
 
-4. Then activate the virutal environment by executing this command:
+4. Then activate the virtual environment by executing this command:
 ###################################################################
 
 .. code-block::
@@ -233,14 +258,14 @@ Execute the following at the project root:
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream.py -s
+    python ./demo/show_stream.py -s
 
 
 Or
 
 .. code-block::
 
-    python ./dnaStreaming/demo/show_stream_async.py -s
+    python ./demo/show_stream_async.py -s
 
 
 Running Docker Demo

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1}/dnaStreaming.egg-info/SOURCES.txt

@@ -3,6 +3,11 @@ MANIFEST.in
 README.rst
 setup.cfg
 setup.py
+demo/__init__.py
+demo/pubsub_stream.py
+demo/show_ha_stream_async.py
+demo/show_stream.py
+demo/show_stream_async.py
 dnaStreaming/__init__.py
 dnaStreaming/config.py
 dnaStreaming/customer_config.json
@@ -12,11 +17,6 @@ dnaStreaming.egg-info/SOURCES.txt
 dnaStreaming.egg-info/dependency_links.txt
 dnaStreaming.egg-info/requires.txt
 dnaStreaming.egg-info/top_level.txt
-dnaStreaming/demo/__init__.py
-dnaStreaming/demo/pubsub_stream.py
-dnaStreaming/demo/show_ha_stream_async.py
-dnaStreaming/demo/show_stream.py
-dnaStreaming/demo/show_stream_async.py
 dnaStreaming/services/__init__.py
 dnaStreaming/services/authentication_service.py
 dnaStreaming/services/availability_service.py

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1}/dnaStreaming.egg-info/requires.txt

@@ -1,3 +1,4 @@
+setuptools>=75.1.0
 googleapis-common-protos>=1.56.4
 google-auth>=2.11.0
 google-cloud-pubsub==2.13.6

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1}/dnaStreaming.egg-info/top_level.txt

@@ -1 +1,2 @@
+demo
 dnaStreaming

{dnastreaming-2.2.1.dev1 → dnastreaming-2.3.1}/setup.py

@@ -6,7 +6,7 @@ this_directory = path.abspath(path.dirname(__file__))
 with open(path.join(this_directory, 'README.rst'), encoding='utf-8') as f:
     long_description = f.read()
 
-VERSION = "2.2.1.dev1"
+VERSION = "2.3.1"
 RELEASE_TAG = f"release-{VERSION}"
 
 setup(
@@ -28,6 +28,7 @@ setup(
     include_package_data=True,
 
     install_requires=[
+        'setuptools>=75.1.0',
         'googleapis-common-protos>=1.56.4',
         'google-auth>=2.11.0',
         'google-cloud-pubsub==2.13.6',

dnastreaming-2.2.1.dev1/dnaStreaming/__init__.py

@@ -1,29 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-import os
-import sys
-import logging
-
-
-BASE_DIR = os.path.dirname(__file__)
-
-logging.basicConfig(level=logging.WARN)
-
-logger = logging.getLogger()
-
-log_path = os.path.join(BASE_DIR, 'logs')
-
-print("Will log to: {}".format(log_path))
-
-if not os.path.exists(log_path):
-    os.mkdir(log_path)
-
-fileHandler = logging.FileHandler("{0}/{1}.log".format(log_path, 'dj-dna-streaming-python'))
-
-logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
-fileHandler.setFormatter(logFormatter)
-logger.addHandler(fileHandler)
-
-consoleHandler = logging.StreamHandler(sys.stdout)
-consoleHandler.setFormatter(logFormatter)
-logger.addHandler(consoleHandler)

dnastreaming-2.2.1.dev1/dnaStreaming/demo/pubsub_stream.py

@@ -1,38 +0,0 @@
-import os
-import json
-from google.cloud import pubsub_v1
-from dnaStreaming.listener import Listener
-from dnaStreaming import logger
-
-gcp_project_id = os.getenv('GCP_PROJECT_ID', None)
-gcp_pubsub_topic = os.getenv('GCP_PUBSUB_TOPIC', None)
-gcp_creds = os.getenv('GOOGLE_APPLICATION_CREDENTIALS', None)
-
-if gcp_project_id is not None and gcp_pubsub_topic is not None and gcp_creds is not None:
-    listener = Listener()
-    ps_publisher = pubsub_v1.PublisherClient()
-    topic_path = ps_publisher.topic_path(gcp_project_id, gcp_pubsub_topic)
-    print("\n[ACTIVITY] Sending messages to Pub/Sub topic {} in GCP\n[0]".format(gcp_pubsub_topic), end='')
-
-    def callback(message, subscription_id, file_handle=None):
-        callback.counter += 1
-        logger.info("Processing article with ID: {}".format(message['an']))
-        m_data = json.dumps(message, ensure_ascii=False).encode("utf-8")
-        ps_publisher.publish(topic_path, data=m_data)
-        logger.info("Sent to GCP Pub/Sub, article with ID: {}".format(message['an']))
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-        return True
-
-    callback.counter = 0
-    listener.listen(callback)
-else:
-    print("[ERROR]: Required ENV variables not set")
-    if gcp_project_id is None:
-        print("    - GCP_PROJECT_ID: GCP Project ID")
-    if gcp_pubsub_topic is None:
-        print("    - GCP_PUBSUB_TOPIC: GCP Pub/Sub Topic Name")
-    if gcp_creds is None:
-        print("    - GOOGLE_APPLICATION_CREDENTIALS: Path to Service Account JSON Credentials File")

dnastreaming-2.2.1.dev1/dnaStreaming/demo/show_ha_stream_async.py

@@ -1,36 +0,0 @@
-import os
-from time import sleep
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-max_secs = 1000
-print("\n[ACTIVITY] Receiving messages (ASYNC) for {} seconds...\n[0]".format(max_secs), end='')
-
-
-def callback(message, subscription_id):
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, TITLE: {}'.format(subscription_id, callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [SUBSCRIPTION]: {} [MSG] [{}]: AN: {}, *** DELETE ***'.format(subscription_id, callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-listener_controller = listener.listen_async_ha(callback)
-
-# Stop receiving messages after 5 seconds
-for count in range(0, max_secs):
-    sleep(1)
-
-if listener_controller.listener_is_running():
-    listener_controller.stop_listener()
-
-print("stop receiving messages")

dnastreaming-2.2.1.dev1/dnaStreaming/demo/show_stream.py

@@ -1,30 +0,0 @@
-import os
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-print("\n[ACTIVITY] Receiving messages (SYNC)...\n[0]", end='')
-
-
-def callback(message, subscription_id):
-    # Replace the code below with the custom operations you'd like to apply to each article (message)
-    # like writing to a file, database, or pushing the message to a separate message broker system.
-    # This demo only prints some message details.
-    # Subscription id parameter is provided and can be used for auditing purposes.
-
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [MSG] [{}]: AN: {}, TITLE: {}'.format(callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [MSG] [{}]: AN: {}, *** DELETE ***'.format(callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-listener.listen(callback)

dnastreaming-2.2.1.dev1/dnaStreaming/demo/show_stream_async.py

@@ -1,36 +0,0 @@
-import os
-from time import sleep
-from dnaStreaming.listener import Listener
-
-listener = Listener()
-quiet_demo = os.getenv('QUIET_DEMO', "false") == "true"
-max_secs = 5
-print("\n[ACTIVITY] Receiving messages (ASYNC) for {} seconds...\n[0]".format(max_secs), end='')
-
-
-def callback(message, subscription_id):
-    callback.counter += 1
-    if not quiet_demo:
-        if message['action'] != 'del':
-            print('[INFO] [MSG] [{}]: AN: {}, TITLE: {}'.format(callback.counter, message['an'], message['title']))
-        else:
-            print('[INFO] [MSG] [{}]: AN: {}, *** DELETE ***'.format(callback.counter, message['an']))
-    else:
-        if callback.counter % 10 == 0:
-            print('[{}]'.format(callback.counter), end='')
-        else:
-            print('.', end='')
-    return True
-
-
-callback.counter = 0
-future = listener.listen_async(callback)
-
-# Stop receiving messages after 5 seconds
-for count in range(0, max_secs):
-    sleep(1)
-
-if future.running():
-    future.cancel()
-
-print("stop receiving messages")