tom-alertstreams 0.0.0__tar.gz

tom_alertstreams-0.0.0/PKG-INFO

@@ -0,0 +1,187 @@
+Metadata-Version: 2.3
+Name: tom-alertstreams
+Version: 0.0.0
+Summary: Reusable TOMToolkit app for listening to kafka streams.
+License: GPL-3.0-only
+Keywords: tomtoolkit,astronomy,astrophysics,cosmology,science
+Author: TOM Toolkit Project
+Author-email: tomtoolkit-maintainers@lco.global
+Maintainer: Joey Chatelain
+Maintainer-email: jchate6@gmail.com
+Requires-Python: >=3.9.0,<3.13
+Classifier: Environment :: Web Environment
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 4.1
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Dist: gcn-kafka (>=0.3,<1.0)
+Requires-Dist: hop-client (>=0.10,<1.0)
+Requires-Dist: psycopg2-binary (>=2.9,<3.0)
+Requires-Dist: tomtoolkit (>=2.22,<3.0)
+Description-Content-Type: text/markdown
+
+# tom-alertstreams
+
+`tom-alertstreams` is a reusable TOM Toolkit app for listening to kafka streams.
+
+`tom-alertstreams` provides a management command, `readstreams`. There are no `urlpatterns`,
+no Views, and no templates. The `readstreams` management command reads the `settings.py` `ALERT_STREAMS`
+configuration and starts listening to each configured Kafka stream. It is not expected
+to return, and is intended to run along side your TOM's server component. The `ALERT_STREAMS`
+configuration (see below) tells `readstreams` what streams to access, how to access them,
+what topics to listen to, and what to do with messages that arrive on a given topic.
+
+## Installation
+
+1. Install the package into your TOM environment:
+    ```bash
+    pip install tom-alertstreams
+   ```
+
+2. In your project `settings.py`, add `tom_alertstreams` to your `INSTALLED_APPS` setting:
+
+    ```python
+    INSTALLED_APPS = [
+        ...
+        'tom_alertstreams',
+    ]
+    ```
+
+At this point you can verify the installation by running `./manage.py` to list the available
+management commands and see
+
+   ```bash
+   [tom_alertstreams]
+       readstreams
+   ```
+in the output.
+
+## Configuration
+
+Each Kafka stream that your TOM listens to (via `readstreams`) will have a configuration dictionary
+in your `settings.py` `ALERT_STREAMS`. `ALERT_STREAMS` is a list of configuration dictionaries, one
+dictionary for each Kafka stream. Here's an example `ALERT_STREAMS` configuration for two Kafka streams:
+[SCiMMA Hopskotch](https://scimma.org/hopskotch.html) and
+[GCN Classic over Kafka](https://gcn.nasa.gov/quickstart).
+
+```python
+ALERT_STREAMS = [
+    {
+        'ACTIVE': True,
+        'NAME': 'tom_alertstreams.alertstreams.hopskotch.HopskotchAlertStream',
+        'OPTIONS': {
+            'URL': 'kafka://kafka.scimma.org/',
+            # The hop-client requires that the GROUP_ID prefix match the SCIMMA_AUTH_USERNAME
+            'GROUP_ID': os.getenv('SCIMMA_AUTH_USERNAME', "") + '-' + 'uniqueidforyourapp12345',
+            'USERNAME': os.getenv('SCIMMA_AUTH_USERNAME', None),
+            'PASSWORD': os.getenv('SCIMMA_AUTH_PASSWORD', None),
+            'TOPIC_HANDLERS': {
+                'sys.heartbeat': 'tom_alertstreams.alertstreams.hopskotch.heartbeat_handler',
+                'tomtoolkit.test': 'tom_alertstreams.alertstreams.hopskotch.alert_logger',
+                'hermes.test': 'tom_alertstreams.alertstreams.hopskotch.alert_logger',
+                'hermes.*': 'regex match public topics here, requires * handler to be defined'
+                '*': 'default_handler_here'
+            },
+        },
+    },
+    {
+        'ACTIVE': True,
+        'NAME': 'tom_alertstreams.alertstreams.gcn.GCNClassicAlertStream',
+        # The keys of the OPTIONS dictionary become (lower-case) properties of the AlertStream instance.
+        'OPTIONS': {
+            # see https://github.com/nasa-gcn/gcn-kafka-python#to-use for configuration details.
+            'GCN_CLASSIC_CLIENT_ID': os.getenv('GCN_CLASSIC_CLIENT_ID', None),
+            'GCN_CLASSIC_CLIENT_SECRET': os.getenv('GCN_CLASSIC_CLIENT_SECRET', None),
+            'DOMAIN': 'gcn.nasa.gov',  # optional, defaults to 'gcn.nasa.gov'
+            'CONFIG': {  # optional
+                # 'group.id': 'tom_alertstreams-my-custom-group-id',
+                # 'auto.offset.reset': 'earliest',
+                # 'enable.auto.commit': False
+            },
+            'TOPIC_HANDLERS': {
+                'gcn.classic.text.LVC_INITIAL': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+                'gcn.classic.text.LVC_PRELIMINARY': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+                'gcn.classic.text.LVC_RETRACTION': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+            },
+        },
+    }
+]
+```
+
+The configuration dictionary for each `AlertStream` subclass will contain these key-value pairs:
+* `ACTIVE`: Boolean which tells `readstreams` to access this stream. Should be `True`, unless you want to
+keep a configuration dictionary, but ignore the stream.
+* `NAME`: The name of the `AlertStream` subclass that implements the interface to this Kafka stream. `tom_alertstreams`
+will provide `AlertStream` subclasses for major astromical Kafka streams. See below for instructions on Subclassing
+the `AlertStream` base class.
+* `OPTIONS`: A dictionary of key-value pairs specific to the`AlertStream` subclass given by `NAME`. The doc string for
+the `AlertStream` subclass should document what is expected. Typically, a URL, authentication information, and a
+dictionary, `TOPIC_HANDLERS`, will be required. See "Subclassing `AlertStream`" below. The `AlertStream` subclass will
+convert the key-value pairs of the `OPTIONS` dictionary into properties (and values) of the `AlertStream` subclass
+instance.
+  * The hopskotch alert stream supports a wildcard of `*` for an alert handler topic name. If specified, ALL public topics will be subscribed and use that handler function. A directly specified topic handler will always be used before the `*` handler for any topic that is covered twice. 
+
+### Getting Kafka Stream Credentials
+As part of your `OPTIONS` for each Kafka stream, you need to configure access credentials. Visit these links
+to get credentials for [Hopskotch](https://hop.scimma.org/) and [GCN Classic over Kafka](https://gcn.nasa.gov/quickstart).
+Set the environment variables with the username and passwords obtained. Do not check them in to your code repository.
+
+
+## Alert Handling
+
+Assuming that an `AlertStream` subclass exists for the Kafka stream of interest,
+the keys of the `TOPIC_HANDLERS` dictionary are the topics that will be subscribed to. The values
+of the `TOPIC_HANDLERS` dictionary specify alert handling methods that will be imported and called
+for each alert recieved on that topic. An example is provided,
+`tom_alerts.alertstreams.alertstream.alert_logger`, which simply logs the alert.
+
+To customize this behaviour according to the needs of your TOM, define an alert handling function for each
+topic that you wish to subscribe to. Your `TOPIC_HANDLERS` dictionary will have a an entry for each topic
+whose key is the topic name and whose value is a string indicating the dot-path to the alert handling function.
+When the `AlertStream` subclass is instanciated, the `OPTIONS` dictionary is read and an `alert_handler`
+dictionary is created. It is keyed by topic name and it's values are the imported callable functions specified by the
+dot-path strings. `readstreams` will call the alert handler for each alert that comes in on the topic. The signiture
+of the alert handling function is specific to the `AlertStream` subclasss.
+
+## Subclassing `AlertStream`
+
+Ideally, As a TOM developer, there is already an `AlertStream`-subclass for the alert stream that you
+want your TOM to listen to. If so, you need only to configure your TOM to use it in  `settings.py`
+`ALERT_STREAMS`. If you must implement your own `AlertStream` subclass, please get in touch. In the meantime, here's a brief outline:
+
+1. Create subclass of `AlertStream`.
+
+2. Create `required_keys` and `allowed_keys` class variables in your `AlertStream`-subclass.
+
+   These are lists of strings refering to the keys of the `OPTIONS` dictionary. The purpose of these is to
+   help TOM developers using your `AlertStream`-subclass with the key-value pairs in their `ALERT_STREAMS`
+  `OPTIONS` configuration dictionary.
+
+3. Implement the `listen()` method.
+
+   This method will be called by the `readstreams` management command and is not expected to return. It
+   should instanciate your consumer, subscribe to the topics configured in `ALERT_STREAMS`, and start
+   consuming. The detail of this will depend on the kafka-client used. See `alertstreams.gcn.listen()`
+   and `alertstreams.hopskotch.listen()` for examples to follow.
+   
+   The loop which consumes messages in your `listen()` method should extract the topic from each message
+   and call `self.alert_handler[topic]()` with the message or message-derived arguments specific to your
+   kafka client. Users of your `AlertStream`-subclass will write these topic-specific alert handling methods
+   and configure them in the `TOPIC_HANLDERS` dictionary of their `ALERT_STREAMS` configuration.
+   The `AlertStream` base class will set up the `alert_handler` dictionary according to your users'
+   configuration. It helps your users to provide an example `alert_hander()` function in your module as
+   an example. (Again, see `alertstreams.gcn.listen()` and `alertstreams.hopskotch.listen()`, their
+   configurations in `settings.py`, and the `alertstreams.gcn.alert_logger()` and
+   `alertstreams.hopskotch.alert_logger() methods, for example).
+

tom_alertstreams-0.0.0/README.md

@@ -0,0 +1,153 @@
+# tom-alertstreams
+
+`tom-alertstreams` is a reusable TOM Toolkit app for listening to kafka streams.
+
+`tom-alertstreams` provides a management command, `readstreams`. There are no `urlpatterns`,
+no Views, and no templates. The `readstreams` management command reads the `settings.py` `ALERT_STREAMS`
+configuration and starts listening to each configured Kafka stream. It is not expected
+to return, and is intended to run along side your TOM's server component. The `ALERT_STREAMS`
+configuration (see below) tells `readstreams` what streams to access, how to access them,
+what topics to listen to, and what to do with messages that arrive on a given topic.
+
+## Installation
+
+1. Install the package into your TOM environment:
+    ```bash
+    pip install tom-alertstreams
+   ```
+
+2. In your project `settings.py`, add `tom_alertstreams` to your `INSTALLED_APPS` setting:
+
+    ```python
+    INSTALLED_APPS = [
+        ...
+        'tom_alertstreams',
+    ]
+    ```
+
+At this point you can verify the installation by running `./manage.py` to list the available
+management commands and see
+
+   ```bash
+   [tom_alertstreams]
+       readstreams
+   ```
+in the output.
+
+## Configuration
+
+Each Kafka stream that your TOM listens to (via `readstreams`) will have a configuration dictionary
+in your `settings.py` `ALERT_STREAMS`. `ALERT_STREAMS` is a list of configuration dictionaries, one
+dictionary for each Kafka stream. Here's an example `ALERT_STREAMS` configuration for two Kafka streams:
+[SCiMMA Hopskotch](https://scimma.org/hopskotch.html) and
+[GCN Classic over Kafka](https://gcn.nasa.gov/quickstart).
+
+```python
+ALERT_STREAMS = [
+    {
+        'ACTIVE': True,
+        'NAME': 'tom_alertstreams.alertstreams.hopskotch.HopskotchAlertStream',
+        'OPTIONS': {
+            'URL': 'kafka://kafka.scimma.org/',
+            # The hop-client requires that the GROUP_ID prefix match the SCIMMA_AUTH_USERNAME
+            'GROUP_ID': os.getenv('SCIMMA_AUTH_USERNAME', "") + '-' + 'uniqueidforyourapp12345',
+            'USERNAME': os.getenv('SCIMMA_AUTH_USERNAME', None),
+            'PASSWORD': os.getenv('SCIMMA_AUTH_PASSWORD', None),
+            'TOPIC_HANDLERS': {
+                'sys.heartbeat': 'tom_alertstreams.alertstreams.hopskotch.heartbeat_handler',
+                'tomtoolkit.test': 'tom_alertstreams.alertstreams.hopskotch.alert_logger',
+                'hermes.test': 'tom_alertstreams.alertstreams.hopskotch.alert_logger',
+                'hermes.*': 'regex match public topics here, requires * handler to be defined'
+                '*': 'default_handler_here'
+            },
+        },
+    },
+    {
+        'ACTIVE': True,
+        'NAME': 'tom_alertstreams.alertstreams.gcn.GCNClassicAlertStream',
+        # The keys of the OPTIONS dictionary become (lower-case) properties of the AlertStream instance.
+        'OPTIONS': {
+            # see https://github.com/nasa-gcn/gcn-kafka-python#to-use for configuration details.
+            'GCN_CLASSIC_CLIENT_ID': os.getenv('GCN_CLASSIC_CLIENT_ID', None),
+            'GCN_CLASSIC_CLIENT_SECRET': os.getenv('GCN_CLASSIC_CLIENT_SECRET', None),
+            'DOMAIN': 'gcn.nasa.gov',  # optional, defaults to 'gcn.nasa.gov'
+            'CONFIG': {  # optional
+                # 'group.id': 'tom_alertstreams-my-custom-group-id',
+                # 'auto.offset.reset': 'earliest',
+                # 'enable.auto.commit': False
+            },
+            'TOPIC_HANDLERS': {
+                'gcn.classic.text.LVC_INITIAL': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+                'gcn.classic.text.LVC_PRELIMINARY': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+                'gcn.classic.text.LVC_RETRACTION': 'tom_alertstreams.alertstreams.alertstream.alert_logger',
+            },
+        },
+    }
+]
+```
+
+The configuration dictionary for each `AlertStream` subclass will contain these key-value pairs:
+* `ACTIVE`: Boolean which tells `readstreams` to access this stream. Should be `True`, unless you want to
+keep a configuration dictionary, but ignore the stream.
+* `NAME`: The name of the `AlertStream` subclass that implements the interface to this Kafka stream. `tom_alertstreams`
+will provide `AlertStream` subclasses for major astromical Kafka streams. See below for instructions on Subclassing
+the `AlertStream` base class.
+* `OPTIONS`: A dictionary of key-value pairs specific to the`AlertStream` subclass given by `NAME`. The doc string for
+the `AlertStream` subclass should document what is expected. Typically, a URL, authentication information, and a
+dictionary, `TOPIC_HANDLERS`, will be required. See "Subclassing `AlertStream`" below. The `AlertStream` subclass will
+convert the key-value pairs of the `OPTIONS` dictionary into properties (and values) of the `AlertStream` subclass
+instance.
+  * The hopskotch alert stream supports a wildcard of `*` for an alert handler topic name. If specified, ALL public topics will be subscribed and use that handler function. A directly specified topic handler will always be used before the `*` handler for any topic that is covered twice. 
+
+### Getting Kafka Stream Credentials
+As part of your `OPTIONS` for each Kafka stream, you need to configure access credentials. Visit these links
+to get credentials for [Hopskotch](https://hop.scimma.org/) and [GCN Classic over Kafka](https://gcn.nasa.gov/quickstart).
+Set the environment variables with the username and passwords obtained. Do not check them in to your code repository.
+
+
+## Alert Handling
+
+Assuming that an `AlertStream` subclass exists for the Kafka stream of interest,
+the keys of the `TOPIC_HANDLERS` dictionary are the topics that will be subscribed to. The values
+of the `TOPIC_HANDLERS` dictionary specify alert handling methods that will be imported and called
+for each alert recieved on that topic. An example is provided,
+`tom_alerts.alertstreams.alertstream.alert_logger`, which simply logs the alert.
+
+To customize this behaviour according to the needs of your TOM, define an alert handling function for each
+topic that you wish to subscribe to. Your `TOPIC_HANDLERS` dictionary will have a an entry for each topic
+whose key is the topic name and whose value is a string indicating the dot-path to the alert handling function.
+When the `AlertStream` subclass is instanciated, the `OPTIONS` dictionary is read and an `alert_handler`
+dictionary is created. It is keyed by topic name and it's values are the imported callable functions specified by the
+dot-path strings. `readstreams` will call the alert handler for each alert that comes in on the topic. The signiture
+of the alert handling function is specific to the `AlertStream` subclasss.
+
+## Subclassing `AlertStream`
+
+Ideally, As a TOM developer, there is already an `AlertStream`-subclass for the alert stream that you
+want your TOM to listen to. If so, you need only to configure your TOM to use it in  `settings.py`
+`ALERT_STREAMS`. If you must implement your own `AlertStream` subclass, please get in touch. In the meantime, here's a brief outline:
+
+1. Create subclass of `AlertStream`.
+
+2. Create `required_keys` and `allowed_keys` class variables in your `AlertStream`-subclass.
+
+   These are lists of strings refering to the keys of the `OPTIONS` dictionary. The purpose of these is to
+   help TOM developers using your `AlertStream`-subclass with the key-value pairs in their `ALERT_STREAMS`
+  `OPTIONS` configuration dictionary.
+
+3. Implement the `listen()` method.
+
+   This method will be called by the `readstreams` management command and is not expected to return. It
+   should instanciate your consumer, subscribe to the topics configured in `ALERT_STREAMS`, and start
+   consuming. The detail of this will depend on the kafka-client used. See `alertstreams.gcn.listen()`
+   and `alertstreams.hopskotch.listen()` for examples to follow.
+   
+   The loop which consumes messages in your `listen()` method should extract the topic from each message
+   and call `self.alert_handler[topic]()` with the message or message-derived arguments specific to your
+   kafka client. Users of your `AlertStream`-subclass will write these topic-specific alert handling methods
+   and configure them in the `TOPIC_HANLDERS` dictionary of their `ALERT_STREAMS` configuration.
+   The `AlertStream` base class will set up the `alert_handler` dictionary according to your users'
+   configuration. It helps your users to provide an example `alert_hander()` function in your module as
+   an example. (Again, see `alertstreams.gcn.listen()` and `alertstreams.hopskotch.listen()`, their
+   configurations in `settings.py`, and the `alertstreams.gcn.alert_logger()` and
+   `alertstreams.hopskotch.alert_logger() methods, for example).

tom_alertstreams-0.0.0/pyproject.toml

@@ -0,0 +1,75 @@
+[project]
+name = "tom-alertstreams"
+description = "Reusable TOMToolkit app for listening to kafka streams."
+authors = [
+    {name = "TOM Toolkit Project", email = "tomtoolkit-maintainers@lco.global"},
+    {name = "Lindy Lindstrom", email = "llindstrom@lco.global"}
+]
+maintainers = [
+    {name = "Joey Chatelain", email = "jchate6@gmail.com"},
+    {name = "William Lindstrom", email = "llindstrom@lco.global"},
+]
+license = "GPL-3.0-only"
+readme = "README.md"
+repository = "https://github.com/TOMToolkit/tom-alertstreams"
+keywords = ["tomtoolkit", "astronomy", "astrophysics", "cosmology", "science"]
+classifiers = [
+    "Environment :: Web Environment",
+    "Framework :: Django",
+    "Framework :: Django :: 4.1",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3 :: Only",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Astronomy",
+    "Topic :: Scientific/Engineering :: Physics"
+]
+dynamic = ["version"]
+requires-python = ">=3.9.0,<3.13"
+packages = [
+    { include = "tom_alertstreams" }
+]
+dependencies = [
+    "tomtoolkit >=2.22,<3.0",
+    "psycopg2-binary >=2.9,<3.0",
+    "gcn-kafka >=0.3,<1.0",
+    "hop-client >=0.10,<1.0"
+]
+
+[tool.setuptools_scm]
+
+[tool.poetry]
+version = "0.0.0" # version supplied by poetry-dynamic-versioning
+[tool.poetry.group.test.dependencies]
+coverage = "^7.6.1"
+factory_boy = "^3.1.0"
+pytest = "^8.3.2"
+
+[tool.poetry.requires-plugins]
+poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
+
+[tool.poetry-dynamic-versioning]
+enable = true
+vcs = "git"
+style = "pep440"
+# the default pattern regex makes the 'v' manditory
+# this pattern modifies the default regex in order to make the 'v' optional
+# ('v' becomes '[v]?' meaning a single v, [v], and ? means optional)
+pattern = "(?x)^[v]?((?P<epoch>\\d+)!)?(?P<base>\\d+(\\.\\d+)*)([-._]?((?P<stage>[a-zA-Z]+)[-._]?(?P<revision>\\d+)?))?(\\+(?P<tagged_metadata>.+))?$"
+
+# substitute version not only in pyproject.toml (which the config block above does)
+# but also the __version__.py file (using the default value of the files property).
+[tool.poetry-dynamic-versioning.substitution]
+
+[build-system]
+requires = [
+    "poetry-core>=1.0.0",
+    "poetry-dynamic-versioning >=1.0.0, <2.0.0"]
+build-backend = "poetry.core.masonry.api"

tom_alertstreams-0.0.0/tom_alertstreams/__init__.py

@@ -0,0 +1,2 @@
+# this is a placeholder for poetry-dynamic-versioning (see pyproject.toml)
+__version__ = "0.0.0"

tom_alertstreams-0.0.0/tom_alertstreams/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

tom_alertstreams-0.0.0/tom_alertstreams/alertstreams/alertstream.py

@@ -0,0 +1,108 @@
+import abc
+import logging
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.utils.module_loading import import_string
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+def get_default_alert_streams():
+    """Return the AlertStreams configured in settings.py
+    """
+    try:
+        alert_streams = get_alert_streams(settings.ALERT_STREAMS)
+    except AttributeError as err:
+        raise ImproperlyConfigured(err)
+
+    return alert_streams
+
+
+def get_alert_streams(alert_stream_configs: list):
+    """Return the AlertStreams configured in the given alert_stream_configs
+    (a list of configuration dictionaries )
+
+    Use get_default_alert_streams() if you want the AlertStreams configured in settings.py.
+    """
+    alert_streams = []  # build and return this list of AlertStream subclass instances
+    for alert_stream_config in alert_stream_configs:
+        if not alert_stream_config.get('ACTIVE', True):
+            logger.debug(f'get_alert_streams - ignoring inactive stream: {alert_stream_config["NAME"]}')
+            continue  # skip configs that are not active; default to ACTIVE
+        try:
+            klass = import_string(alert_stream_config['NAME'])
+        except ImportError:
+            msg = (
+                f'The module (the value of the NAME key): {alert_stream_config["NAME"]} could not be imported. '
+                f'Check your ALERT_STREAMS setting.'
+            )
+            raise ImproperlyConfigured(msg)
+
+        alert_stream: AlertStream = klass(**alert_stream_config.get("OPTIONS", {}))
+        alert_streams.append(alert_stream)
+
+    return alert_streams
+
+
+class AlertStream(abc.ABC):
+    """Base class for specific alert streams like Hopskotch, GCNClassic, etc.
+
+    * kwargs to __init__ is the OPTIONS dictionary defined in ALERT_STREAMS configuration
+    dictionary (for example, see settings.py).
+    * allowed_keys and required_keys should be defined as class properties in subclasses.
+    * The allowed_keys are turned into instance properties in __init__.
+    * Missing required_keys result in an ImproperlyConfigured Django exception.
+
+
+    To implmement subclass:
+    1. define allowed_key, required_keys as class variables
+       <say what these do: used with OPTIONS dict in ALERT_STREAMS config dict>
+    2. implement listen()
+       this method probably doesn't return
+    3. write your alert_handlers. which proably take and alert do something.
+       The HopskotchAlertStream.listen() method defines an 'alert_handlers' dictionary keyed by
+       alert topic with callable values (i.e call this method with alerts from this topic).
+       The GCNClassicAlertStream.listen() is another example.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__()
+
+        # filter the kwargs by allowed keys and add them as properties to AlertStream instance
+        self.__dict__.update((k.lower(), v) for k, v in kwargs.items() if k in self.allowed_keys)
+
+        missing_keys = set(self.required_keys) - set(kwargs.keys())
+        if missing_keys:
+            msg = (
+                f'The following required keys are missing from the configuration OPTIONS of '
+                f'{self._get_stream_classname()}: {list(missing_keys)} ; '
+                f'These keys were found: {list(kwargs.keys())} ; '
+                f'Check your ALERT_STREAMS setting.'
+            )
+            raise ImproperlyConfigured(msg)
+
+        self.alert_handler = self._process_topic_handlers()
+
+    def _get_stream_classname(self) -> str:
+        return type(self).__qualname__
+
+    def _process_topic_handlers(self):
+        """ convert the TOPIC_HANDLERS values in to callable functions in
+        the returned message_handler dictionary. (keyed by topic, value is callable)
+        """
+        alert_handler = {}
+        for topic, callable_string in self.topic_handlers.items():
+            # convert string from TOPIC_HANDLERS in to a callable function in
+            # the message_handler dictionary (both keyed by topic string)
+            alert_handler[topic] = import_string(callable_string)
+        return alert_handler
+
+    @abc.abstractmethod
+    def listen(self):
+        """Listen at the steam and dispatch alerts to handlers. Subclass extentions of
+        this method are not expected to return. See hopskotch.py and gcn.py for example
+        implementations.
+        """
+        pass

tom_alertstreams-0.0.0/tom_alertstreams/alertstreams/gcn.py

@@ -0,0 +1,69 @@
+import logging
+
+from gcn_kafka import Consumer
+
+from tom_alertstreams.alertstreams.alertstream import AlertStream
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+
+class GCNClassicAlertStream(AlertStream):
+    """
+
+    Pre-requisite: visit gcn.nasa.gov and sign-up to get your client_id and
+    client_secret.
+    """
+    # Upon __init__, the AlertStream base class creates instance properties from
+    # the settings OPTIONS dictionary, converting the keys to lowercase.
+    required_keys = ['GCN_CLASSIC_CLIENT_ID', 'GCN_CLASSIC_CLIENT_SECRET', 'TOPIC_HANDLERS']
+    allowed_keys = ['GCN_CLASSIC_CLIENT_ID', 'GCN_CLASSIC_CLIENT_SECRET', 'TOPIC_HANDLERS', 'DOMAIN', 'CONFIG']
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # properties have been created from the OPTIONS dicttionary
+
+    def listen(self):
+        super().listen()
+
+        consumer = Consumer(client_id=self.gcn_classic_client_id,
+                            client_secret=self.gcn_classic_client_secret,
+                            domain=self.domain,
+                            config=self.config,
+                            )
+
+        consumer.subscribe(list(self.topic_handlers.keys()))
+
+        # logger.debug(f'Here is a list of the available topics for {self.domain}')
+        # for topic in consumer.list_topics().topics:
+        #     logger.debug(f'topic: {topic}')
+
+        # what is a cimpl.Message?, cimpl.KafkaError?
+        # see https://docs.confluent.io/4.1.1/clients/confluent-kafka-python/index.html#message
+        while True:
+            for alert in consumer.consume():
+                kafka_error = alert.error()  # cimpl.KafkaError
+                if kafka_error is None:
+                    # no error, so call the alert handler
+                    topic = alert.topic()
+                    try:
+                        self.alert_handler[topic](alert)
+                    except KeyError as err:
+                        logger.error(f'alert from topic {topic} received but no handler defined. err: {err}')
+                else:
+                    logger.error(f'GCNClassicAlertStream KafkaError: {kafka_error.name()}: {kafka_error.str()}')
+        consumer.close()
+
+
+def alert_logger(alert):
+    """Example alert handler for GCN Classic over Kafka
+
+    This alert handler simply logs the topic and value of the cimpl.Message instance.
+
+    See https://docs.confluent.io/4.1.1/clients/confluent-kafka-python/index.html#message
+    for cimpl.Message details.
+    """
+    logger.info(f'gcn.alert_logger alert.topic(): {alert.topic()}')
+    logger.info(f'gcn.alert_logger alert.value(): {alert.value()}')
+

tom_alertstreams-0.0.0/tom_alertstreams/alertstreams/hopskotch.py

@@ -0,0 +1,164 @@
+from datetime import datetime, timezone
+import logging
+import re
+import uuid
+import traceback
+
+from django.utils import timezone as tz
+from django.core.exceptions import ImproperlyConfigured
+
+from hop import Stream
+from hop.auth import Auth
+from hop.models import JSONBlob
+from hop.io import Metadata, StartPosition, list_topics
+
+from tom_alertstreams.alertstreams.alertstream import AlertStream
+
+logger = logging.getLogger(__name__)
+
+
+class HopskotchAlertStream(AlertStream):
+    """
+    """
+    required_keys = ['URL', 'GROUP_ID', 'USERNAME', 'PASSWORD', 'TOPIC_HANDLERS']
+    allowed_keys = ['URL', 'GROUP_ID', 'USERNAME', 'PASSWORD', 'TOPIC_HANDLERS']
+    PUBLIC_TOPIC_CHECK_INTERVAL = 300  # Seconds between checking for new public topics
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        # the following methods may fail if improperly configured.
+        # So, do them now to catch any errors, before listen() is spawned in it's own Process.
+        self.public_topics = self.get_all_public_topics()
+        self.stream_url = self.get_stream_url()
+        self.stream = self.get_stream()
+
+    def get_all_public_topics(self) -> list[str]:
+        """Returns the up-to-date list of Topic names to consume.
+
+        Use the saved options to repeatedly construct the topic list, and
+        keep it in sync with the publicaly_readable topics from SCiMMA Auth.
+
+        The Topic list is a combination of the
+          a. the publicly_readable Topics from SCiMMA Auth
+          b. any topics supplied on the command line via -T, --topic
+        """
+        hop_auth = Auth(self.username, self.password)
+        logger.info('getting publicly_readable topics from SCiMMA Auth.')
+        # use the hop-client to ask Kafka directly for the topics since SCiMMA Auth can be out of sync
+        # include only topics that a) contain a '.'; b) don't start with '__' (excludes __consumer_offsets)
+        publicly_readable_topics = [topic for topic in list_topics(self.url, hop_auth).keys()
+                                    if not (topic.startswith('__') and (topic.count('.')==0))]
+        logger.info(f'publicly_readable_topics: {publicly_readable_topics}')
+
+        return publicly_readable_topics
+
+    def get_stream_url(self) -> str:
+        """For Hopskotch, topics are specified on the url. So, this
+        method gets a base url (from super) and then adds topics to it.
+
+        Hopskotch (hop.io) requires at least one topic to be specified.
+
+        You might not need a method like this if your Kafka client provides
+        alternative ways to subscribe to a topic. For example, the gcn_kafka.Consumer
+        class provides a 'substribe([list of topics])' method. (see gcn.py).
+        """
+        logger.debug(f'HopskotchAlertStream.get_stream_url topics: {list(self.topic_handlers.keys())}')
+        if self.topic_handlers == {}:
+            msg = 'Hopskotch requires at least one topic to open the stream. Check ALERT_STREAMS in settings.py'
+            raise ImproperlyConfigured(msg)
+
+        base_stream_url = self.url
+
+        # if not present, add trailing slash to base_stream url
+        # so, comma-separated topics can be appeneded.
+        if base_stream_url[-1] != '/':
+            base_stream_url += '/'
+
+        # append comma-separated topics to base URL
+        specified_topics = list(self.topic_handlers.keys())
+        if '*' in specified_topics:
+            # Add all public topics if a asterisk is set in the topic_handlers
+            specified_topics = list(set(specified_topics + self.public_topics))
+        # Also remove topics with wildcards in them
+        specified_topics = [topic for topic in specified_topics if not '*' in topic]
+
+        topics = ','.join(specified_topics)  # 'topic1,topic2,topic3'
+        hopskotch_stream_url = base_stream_url + topics
+
+        logger.debug(f'HopskotchAlertStream.get_stream_url url: {hopskotch_stream_url}')
+        return hopskotch_stream_url
+
+    def get_stream(self, start_position=StartPosition.LATEST) -> Stream:
+        hop_auth = Auth(self.username, self.password)
+
+        # TODO: allow StartPosition to be set from OPTIONS configuration dictionary
+        stream = Stream(auth=hop_auth, start_at=start_position)
+        return stream
+
+    def listen(self):
+        super().listen()
+        # TODO: alternatively, WARN upon OPTIONS['topics'] extries that don't have
+        # handlers in the alert_handler. (i.e they've configured a topic subscription
+        # without providing a handler for the topic. So, warn them).
+        last_check_time = tz.now()
+        while True:
+            try:
+                logger.info(f'HopskotchAlertStream.listen opening stream: {self.stream_url} with group_id: {self.group_id}')
+                with self.stream.open(self.stream_url, 'r', group_id=self.group_id) as src:
+                    for alert, metadata in src.read(metadata=True):
+                        # type(gcn_circular) is <hop.models.GNCCircular>
+                        # type(metadata) is <hop.io.Metadata>
+                        if metadata.topic in self.alert_handler:
+                            # TODO: should probably use *args, **kwargs to pass unknow number of arguments
+                            self.alert_handler[metadata.topic](alert, metadata)
+                        elif '*' in self.alert_handler:
+                            # First check all wildcard topics to see if they will match this topic
+                            matched_handler = False
+                            for topic in self.alert_handler.keys():
+                                if topic != '*' and '*' in topic and re.match(topic, metadata.topic):
+                                    self.alert_handler[topic](alert, metadata)
+                                    matched_handler = True
+                                    break
+                            # If nothing matched, fall back to default public topic handler
+                            if not matched_handler:
+                                self.alert_handler['*'](alert, metadata)
+                        else:
+                            logger.error(f'alert from topic {metadata.topic} received but no handler defined.')
+                            # TODO: should define a default handler for all unhandeled topics
+                        if (tz.now() - last_check_time).total_seconds() > self.PUBLIC_TOPIC_CHECK_INTERVAL:
+                            last_check_time = tz.now()
+                            public_topics = self.get_all_public_topics()
+                            if set(public_topics) != set(self.public_topics):
+                                logger.info(f"New public topics found, restarting hop stream")
+                                self.public_topics = public_topics
+                                self.stream_url = self.get_stream_url()
+                                break
+            except Exception as ex:
+                logger.error(f'HopskotchAlertStream.listen: {ex}')
+                logger.error(traceback.format_exc())  # Show the traceback so we have a chance of figuring out what is breaking
+
+def heartbeat_handler(heartbeat: JSONBlob, metadata: Metadata):
+    """Example alert handler for HopskotchAlertStream sys.heartbeat topic.
+
+    Note that HopskotchAlertStream.listen() method knows that Hopskotch alerts come with
+    both alert and metadata. So, the alert_handler methods have a signiture (taking both
+    as arguments) specific to this stream.
+    """
+    content: dict = heartbeat.content  # see hop_client reatthedocs
+    timestamp = datetime.fromtimestamp(content["timestamp"] / 1e6, tz=timezone.utc)
+    if heartbeat.content['count'] % 300 == 0:
+        # mod 300 just for convenience so as not to flood logger
+        logging.info(f'{timestamp.isoformat()} heartbeat.content dict: {heartbeat.content}. metadata: {metadata}')
+
+
+def alert_logger(alert: JSONBlob, metadata: Metadata):
+    """Example alert handler. The method signsture is specific to Hopskotch alerts.
+    """
+    # search the header (list of tuples) for a UUID-tuple (keyed by '_id')
+    # eg. ('_id', b'$\xd6oGmVM\xed\x97\xe7|\x1c\x8f\x11V\xe9')
+    alert_uuid_tuple = next((item for item in metadata.headers if item[0] == '_id'), None)
+    if alert_uuid_tuple:
+        alert_uuid = uuid.UUID(bytes=alert_uuid_tuple[1])
+    else:
+        # in this case the alert was probably published with hop-client<0.8.0
+        alert_uuid = None
+    logger.info(f'Alert (uuid={alert_uuid}) received on topic {metadata.topic}: {alert};  metatdata: {metadata}')

tom_alertstreams-0.0.0/tom_alertstreams/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class TomAlertstreamsConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'tom_alertstreams'

tom_alertstreams-0.0.0/tom_alertstreams/management/commands/hoptestpub.py

@@ -0,0 +1,40 @@
+import datetime
+import logging
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand
+
+from tom_alertstreams.alertstreams.alertstream import get_default_alert_streams
+from tom_alertstreams.alertstreams.hopskotch import HopskotchAlertStream
+
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.INFO)
+
+
+class Command(BaseCommand):
+    help = 'Publish a timestamped test message to Hopskotch tomtoolkit.test topic.'
+
+    def handle(self, *args, **options):
+        logger.debug(f'hoptestpub.Command.handle() args: {args}')
+        logger.debug(f'hoptestpub.Command.handle() options: {options}')
+
+        try:
+            alert_streams = get_default_alert_streams()
+            # extract the HopskotchAlertStream
+            hopskotch_alert_stream = next(stream for stream in alert_streams if isinstance(stream, HopskotchAlertStream))
+
+        except ImproperlyConfigured as ex:
+            logger.error(f'{ex.__class__.__name__}: Configure alert streams in settings.py ALERT_STREAMS: {ex}')
+            exit(1)
+
+        stream = hopskotch_alert_stream.get_stream()
+        topic = 'tomtoolkit.test'  # SCiMMA Admin topic permissions for Credential are assumed to have been set up
+
+        with stream.open(hopskotch_alert_stream.url+topic, "w") as s:
+            s.write({
+                'created': datetime.datetime.utcnow().isoformat(),
+                'created-by': 'tom-alertstreams hoptestpub.py'
+             })
+
+        logger.info('hoptestpub Command.handle() returning...')

tom_alertstreams-0.0.0/tom_alertstreams/management/commands/readstreams.py

@@ -0,0 +1,38 @@
+import logging
+from threading import Thread
+
+from django.core.exceptions import ImproperlyConfigured
+from django.core.management.base import BaseCommand
+
+from tom_alertstreams.alertstreams.alertstream import get_default_alert_streams
+
+
+logger = logging.getLogger(__name__)
+# logger.setLevel(logging.DEBUG)
+logger.setLevel(logging.INFO)
+
+
+class Command(BaseCommand):
+    help = 'Consume alerts from the alert streams configured in the settings.py ALERT_STREAMS'
+
+    def handle(self, *args, **options):
+        logger.debug(f'readstreams.Command.handle() args: {args}')
+        logger.debug(f'readstreams.Command.handle() options: {options}')
+
+        try:
+            alert_streams = get_default_alert_streams()
+        except ImproperlyConfigured as ex:
+            logger.error(f'{ex.__class__.__name__}: Configure alert streams in settings.py ALERT_STREAMS: {ex}')
+            exit(1)
+
+        try:
+            # listen to each alert_stream in it's own Thread (sort of at the same time)
+            for alert_stream in alert_streams:
+                t = Thread(target=alert_stream.listen, name=alert_stream._get_stream_classname())
+                t.start()
+                logger.info((f'read_streams {alert_stream._get_stream_classname()} TID={t.native_id} ; '
+                             f'thread identifier={t.ident}'))
+        except KeyboardInterrupt as msg:
+            logger.info(f'read_streams handling KeyboardInterupt {msg}')
+
+        logger.info('readstreams Command.handle() returning...')

tom_alertstreams-0.0.0/tom_alertstreams/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

tom_alertstreams-0.0.0/tom_alertstreams/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

tom_alertstreams-0.0.0/tom_alertstreams/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.