awsimple 3.6.1__py3-none-any.whl → 3.9.1__py3-none-any.whl

awsimple/__init__.py

@@ -6,6 +6,7 @@ from .dynamodb import DynamoDBAccess, dict_to_dynamodb, DBItemNotFound, DynamoDB
 from .dynamodb import KeyType, aws_name_to_key_type
 from .dynamodb_miv import DynamoDBMIVUI, miv_string, get_time_us, miv_us_to_timestamp
 from .s3 import S3Access, S3DownloadStatus, S3ObjectMetadata, BucketNotFound
-from .sqs import SQSAccess, SQSPollAccess, aws_sqs_long_poll_max_wait_time, aws_sqs_max_messages
+from .sqs import SQSAccess, SQSPollAccess, aws_sqs_long_poll_max_wait_time, aws_sqs_max_messages, get_all_sqs_queues
 from .sns import SNSAccess
+from .pubsub import PubSub
 from .logs import LogsAccess

awsimple/__version__.py

@@ -1,7 +1,7 @@
 __application_name__ = "awsimple"
 __title__ = __application_name__
 __author__ = "abel"
-__version__ = "3.6.1"
+__version__ = "3.9.1"
 __author_email__ = "j@abel.co"
 __url__ = "https://github.com/jamesabel/awsimple"
 __download_url__ = "https://github.com/jamesabel/awsimple"

awsimple/logs.py

@@ -1,22 +1,10 @@
 import time
-import getpass
-import platform
-from functools import lru_cache
 from typing import Union
 from pathlib import Path
 from datetime import datetime
 
-from awsimple import AWSAccess
-
-
-@lru_cache()
-def get_user_name() -> str:
-    return getpass.getuser()
-
-
-@lru_cache()
-def get_computer_name() -> str:
-    return platform.node()
+from .aws import AWSAccess
+from .platform import get_user_name, get_computer_name
 
 
 class LogsAccess(AWSAccess):

awsimple/platform.py

@@ -0,0 +1,20 @@
+from functools import cache
+
+import getpass
+import platform
+
+
+@cache
+def get_user_name() -> str:
+    return getpass.getuser()
+
+
+@cache
+def get_computer_name() -> str:
+    return platform.node()
+
+
+@cache
+def get_node_name() -> str:
+    node_name = f"{get_computer_name()}-{get_user_name()}"  # AWS SNS and SQS names can include hyphens
+    return node_name

awsimple/pubsub.py

@@ -0,0 +1,213 @@
+"""
+pub/sub abstraction on top of AWS SNS and SQS using boto3.
+"""
+
+import time
+from typing import Any, Dict, List, Callable
+from datetime import timedelta
+from multiprocessing import Process, Queue, Event
+from threading import Thread
+from queue import Empty
+from logging import Logger
+import json
+
+from typeguard import typechecked
+from botocore.exceptions import ClientError
+
+from .sns import SNSAccess
+from .sqs import SQSPollAccess, get_all_sqs_queues
+from .dynamodb import _DynamoDBMetadataTable
+from .platform import get_node_name
+from .__version__ import __application_name__
+
+log = Logger(__application_name__)
+
+queue_timeout = timedelta(days=30).total_seconds()
+
+
+@typechecked()
+def remove_old_queues(channel: str) -> list[str]:
+    """
+    Remove old SQS queues that have not been used recently.
+    """
+    removed = []  # type: list[str]
+    if len(channel) < 2:  # avoid deleting all queues
+        log.warning(f"blank channel ({channel=}) - not deleting any queues")
+        return removed
+    for sqs_queue_name in get_all_sqs_queues(channel):
+        sqs_metadata = _DynamoDBMetadataTable(sqs_queue_name)
+        mtime = sqs_metadata.get_table_mtime_f()
+        if mtime is not None and time.time() - mtime > queue_timeout:
+            sqs = SQSPollAccess(sqs_queue_name)
+            try:
+                sqs.delete_queue()
+                log.info(f'deleted "{sqs_queue_name}",{mtime=}')
+            except ClientError:
+                log.info(f'"{sqs_queue_name}" already does not exist,{mtime=}')  # already doesn't exist - this is benign
+            removed.append(sqs_queue_name)
+    return removed
+
+
+@typechecked()
+def _connect_sns_to_sqs(channel_name: str, sqs_queue_name: str, sqs: SQSPollAccess) -> None:
+    """
+    Connect an SQS queue to an SNS topic.
+
+    :param channel_name: SNS topic name.
+    :param sqs_queue_name: SQS queue name.
+    :param sqs: SQSPollAccess instance for the SQS queue.
+    :return: None
+    """
+
+    sqs_arn = sqs.get_arn()
+
+    # Find the topic by name
+    sns = SNSAccess(channel_name)
+    sns.create_topic()
+    topic_arn = sns.get_arn()
+    assert sns.resource is not None
+    topic = sns.resource.Topic(topic_arn)
+
+    # Subscribe queue to topic
+    queue_arn = sqs.get_arn()
+    subscription = topic.subscribe(Protocol="sqs", Endpoint=queue_arn)
+    log.info(f"Subscribed {sqs_queue_name} to topic {topic_arn}. Subscription ARN: {subscription.arn}")
+
+    # Update queue policy to allow SNS -> SQS
+    policy = {
+        "Version": "2012-10-17",
+        "Id": "sns-sqs-subscription-policy",
+        "Statement": [
+            {
+                "Sid": "Allow-SNS-SendMessage",
+                "Effect": "Allow",
+                "Principal": {"Service": "sns.amazonaws.com"},
+                "Action": "sqs:SendMessage",
+                "Resource": sqs_arn,
+                "Condition": {"ArnEquals": {"aws:SourceArn": topic_arn}},
+            }
+        ],
+    }
+    assert sqs.queue is not None
+    sqs.queue.set_attributes(Attributes={"Policy": json.dumps(policy)})
+    log.debug(f"Queue {sqs_queue_name} policy updated to allow topic {topic_arn}.")
+
+
+class _SubscriptionThread(Thread):
+    """
+    Thread to poll SQS for new messages and put them in a queue for the parent process to read.
+    """
+
+    @typechecked()
+    def __init__(self, sqs: SQSPollAccess, new_event) -> None:
+        super().__init__()
+        self._sqs = sqs
+        self.new_event = new_event
+        self.sub_queue = Queue()  # type: Queue[str]
+
+    def run(self):
+        # exit by terminating the parent process
+        while True:
+            messages = self._sqs.receive_messages()  # long poll
+            for message in messages:
+                message = json.loads(message.message)
+                self.sub_queue.put(message["Message"])
+                self.new_event.set()  # notify parent process that a new message is available
+
+
+class PubSub(Process):
+
+    @typechecked()
+    def __init__(self, channel: str, node_name: str = get_node_name(), sub_callback: Callable | None = None) -> None:
+        """
+        Pub and Sub.
+        Create in a separate process to offload from main thread. Also facilitates use of moto mock in tests.
+
+        :param channel: Channel name (SNS topic name).
+        :param node_name: Node name (SQS queue name suffix). Defaults to a combination of computer name and username, but can be passed in for customization and/or testing.
+        :param sub_callback: Optional thread and process safe callback function to be called when a new message is received. The function should accept a single argument, which will be the message as a dictionary.
+        """
+        self.channel = channel
+        self.node_name = node_name  # e.g., computer name
+        self.sub_callback = sub_callback
+
+        self._pub_queue = Queue()  # type: Queue[Dict[str, Any]]
+        self._sub_queue = Queue()  # type: Queue[str]
+
+        self._new_event = Event()  # pub or sub sets this when a new message is available or has been sent
+
+        super().__init__()
+
+    def run(self):
+
+        sqs_prefix = f"{self.channel}-"
+        sqs_queue_name = f"{sqs_prefix}{self.node_name}"
+
+        sns = SNSAccess(self.channel, auto_create=True)
+        sqs_metadata = _DynamoDBMetadataTable(sqs_queue_name)
+
+        sqs = SQSPollAccess(sqs_queue_name)
+        if not sqs.exists():
+            sqs.create_queue()
+            _connect_sns_to_sqs(self.channel, sqs_queue_name, sqs)
+
+        sqs_metadata.update_table_mtime()  # update SQS use time (the existing infrastructure calls it a "table", but we're using it for the SQS queue)
+        remove_old_queues(sqs_prefix)  # clean up old queues
+
+        sqs_thread = _SubscriptionThread(sqs, self._new_event)
+        sqs_thread.start()
+
+        while True:
+
+            # pub
+            try:
+                message = self._pub_queue.get(False)
+                message_string = json.dumps(message)
+                sns.publish(message_string)
+            except Empty:
+                pass
+
+            # sub
+            try:
+                message_string = sqs_thread.sub_queue.get(False)
+                # if a callback is provided, call it, otherwise put the message in the sub queue for later retrieval
+                if self.sub_callback is None:
+                    self._sub_queue.put(message_string)
+                else:
+                    message = json.loads(message_string)
+                    self.sub_callback(message)
+                sqs_metadata.update_table_mtime()
+            except Empty:
+                pass  # no message
+
+            # this helps responsiveness
+            self._new_event.clear()
+            self._new_event.wait(3)  # race conditions can occur, so don't make this wait timeout too long
+
+    @typechecked()
+    def publish(self, message: dict) -> None:
+        """
+        Publish a message.
+
+        :param message: message as a dictionary
+        """
+        self._pub_queue.put(message)
+        self._new_event.set()
+
+    @typechecked()
+    def get_messages(self) -> List[Dict[str, Any]]:
+        """
+        Get all available messages.
+
+        :return: list of messages as dictionaries
+        """
+        messages = []
+        while True:
+            try:
+                message_string = self._sub_queue.get(block=False)
+                message = json.loads(message_string)
+                log.debug(f"{message=}")
+                messages.append(message)
+            except Empty:
+                break
+        return messages

awsimple/sns.py

@@ -2,7 +2,8 @@
 SNS Access
 """
 
-from typing import Union, Dict
+from typing import Union, Dict, Any
+from functools import cache
 
 from typeguard import typechecked
 
@@ -11,7 +12,7 @@ from awsimple import AWSAccess, SQSAccess
 
 class SNSAccess(AWSAccess):
     @typechecked()
-    def __init__(self, topic_name: str, **kwargs):
+    def __init__(self, topic_name: str, auto_create: bool = False, **kwargs):
         """
         SNS Access
 
@@ -20,26 +21,31 @@ class SNSAccess(AWSAccess):
         """
         super().__init__(resource_name="sns", **kwargs)
         self.topic_name = topic_name
+        self.auto_create = auto_create
 
-    def get_topic(self):
+    @cache
+    def get_topic(self) -> Any:
         """
         gets the associated SNS Topic instance
 
-        :param topic_name: topic name
         :return: sns.Topic instance
         """
         topic = None
+        assert self.resource is not None
         for t in self.resource.topics.all():
             if t.arn.split(":")[-1] == self.topic_name:
                 topic = t
+        if self.auto_create and topic is None:
+            self.create_topic()
+            self.auto_create = False  # only do this once
+            topic = self.get_topic()
         return topic
 
-    @typechecked()
+    @cache
     def get_arn(self) -> str:
         """
         get topic ARN from topic name
 
-        :param topic_name: topic name string
         :return: topic ARN
         """
         return self.get_topic().arn
@@ -68,7 +74,7 @@ class SNSAccess(AWSAccess):
         """
         Subscribe to an SNS topic
 
-        :param subscriber: email or SQS queue
+        :param subscriber: email
         :return: subscription ARN
         """
         if isinstance(subscriber, str) and "@" in subscriber:

awsimple/sqs.py

@@ -49,7 +49,7 @@ aws_sqs_max_messages = 10
 
 class SQSAccess(AWSAccess):
     @typechecked()
-    def __init__(self, queue_name: str, immediate_delete: bool = True, visibility_timeout: Union[int, None] = None, minimum_visibility_timeout: int = 0, **kwargs):
+    def __init__(self, queue_name: str, immediate_delete: bool = True, visibility_timeout: Union[int, None] = None, minimum_visibility_timeout: int = 0, auto_create: bool = False, **kwargs):
         """
         SQS access
 
@@ -66,6 +66,7 @@ class SQSAccess(AWSAccess):
         self.immediate_delete = immediate_delete  # True to immediately delete messages
         self.user_provided_timeout = visibility_timeout  # the queue will re-try a message (make it re-visible) if not deleted within this time
         self.user_provided_minimum_timeout = minimum_visibility_timeout  # the timeout will be at least this long
+        self.auto_create = auto_create  # automatically create the queue if it doesn't exist
         self.auto_timeout_multiplier = 10.0  # for automatic timeout calculations, multiply this times the median run time to get the timeout
 
         self.sqs_call_wait_time = 0  # short (0) or long poll (> 0, usually 20)
@@ -80,21 +81,38 @@ class SQSAccess(AWSAccess):
         # We write the history out as a file so don't make this too big. We take the median (for the nominal run time) so make this big enough to tolerate a fair number of outliers.
         self.max_history = 20
 
-    def _get_queue(self):
+        self.was_created = False
+
+    def _get_queue(self) -> Any:
+        """
+        Get the SQS queue instance. If the queue doesn't exist and auto_create is True, it will be created.
+
+        :return: SQS queue instance
+        """
         if self.queue is None:
-            try:
-                queue = self.resource.get_queue_by_name(QueueName=self.queue_name)
-            except self.client.exceptions.QueueDoesNotExist as e:
-                log.debug(f"{self.queue_name},{e=}")
-                queue = None
-            except self.client.exceptions.ClientError as e:
-                error_code = e.response["Error"].get("Code")
-                if "NonExistentQueue" in error_code:
-                    log.debug(f"{self.queue_name},{e=},{error_code=}")
+            queue = None
+            try_count = 0
+            assert self.resource is not None
+            while self.queue is None and try_count < 3:
+                create_queue = False
+                try:
+                    queue = self.resource.get_queue_by_name(QueueName=self.queue_name)
+                except self.client.exceptions.QueueDoesNotExist as e:
+                    if self.auto_create:
+                        create_queue = True
+                    log.debug(f"{self.queue_name},{e=}")
                     queue = None
-                else:
-                    # other errors (e.g. connection errors, etc.)
-                    raise
+                except self.client.exceptions.ClientError as e:
+                    error_code = e.response["Error"].get("Code")
+                    if "NonExistentQueue" in error_code:
+                        log.debug(f"{self.queue_name},{e=},{error_code=}")
+                        queue = None
+                    else:
+                        # other errors (e.g. connection errors, etc.)
+                        raise
+                if create_queue and self.create_queue() is None:
+                    time.sleep(60)  # if we just deleted the queue, we may have to wait 60 seconds before we can re-create it
+                try_count += 1
 
             if queue is not None:
                 # kludge so when moto mocking we return None if it can't get the queue
@@ -119,14 +137,19 @@ class SQSAccess(AWSAccess):
         return p
 
     @typechecked()
-    def create_queue(self) -> str:
+    def create_queue(self) -> str | None:
         """
         create SQS queue
 
-        :return: queue URL
+        :return: queue URL or None if not successful
         """
-        response = self.client.create_queue(QueueName=self.queue_name)
-        url = response.get("QueueUrl", "")
+        try:
+            response = self.client.create_queue(QueueName=self.queue_name)
+            url = response.get("QueueUrl")
+            self.was_created = True
+        except self.client.exceptions.QueueDeletedRecently as e:
+            log.warning(f"{self.queue_name},{e}")  # can happen if a queue was recently deleted, and we try to re-create it too quickly
+            url = None
         return url
 
     def delete_queue(self):
@@ -367,6 +390,30 @@ class SQSAccess(AWSAccess):
 
 
 class SQSPollAccess(SQSAccess):
+    """
+    SQS Access with long polling.
+    """
+
     def __init__(self, queue_name: str, **kwargs):
-        super().__init__(queue_name, **kwargs)
+        super().__init__(queue_name=queue_name, **kwargs)
         self.sqs_call_wait_time = aws_sqs_long_poll_max_wait_time
+
+
+@typechecked()
+def get_all_sqs_queues(prefix: str = "") -> List[str]:
+    """
+    get all SQS queues
+
+
+    :param prefix: prefix to filter queue names (empty string for all queues)
+    :return: list of queue names
+    """
+    queue_names = []
+
+    sqs = AWSAccess("sqs")
+    for queue in list(sqs.resource.queues.all()):
+        queue_name = queue.url.split("/")[-1]
+        if queue_name.startswith(prefix):
+            queue_names.append(queue_name)
+
+    return queue_names

{awsimple-3.6.1.dist-info → awsimple-3.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: awsimple
-Version: 3.6.1
+Version: 3.9.1
 Summary: Simple AWS API for S3, DynamoDB, SNS, and SQS
 Home-page: https://github.com/jamesabel/awsimple
 Download-URL: https://github.com/jamesabel/awsimple
@@ -75,26 +75,28 @@ Full documentation available on [Read the Docs](https://awsimple.readthedocs.io/
 
 ### Features:
 
-- Simple Object Oriented API on top of boto3
+- Simple Object-Oriented API on top of boto3.
+  - Eliminates the need to worry about `clients`, `resources`, `sessions`, and pagination.
 
-- One-line S3 file write, read, and delete
+- Locally cached S3 accesses. Reduces network traffic, AWS costs, and can speed up access.
 
-- Automatic S3 retries
+- `pubsub` functionality (via SNS topics and SQS queues).
 
-- Locally cached S3 accesses
+- DynamoDB full table scans (with local cache option that only rescans if the table has changed).
 
-- True file hashing (SHA512) for S3 files (S3's etag is not a true file hash)
+- True file hashing (SHA512) for S3 files (S3's etag is not a true file hash).
 
-- DynamoDB full table scans (with local cache option)
+- Supports moto mock and localstack. Handy for testing and CI.
 
-- DynamoDB secondary indexes
+- Automatic S3 retries.
 
-- Built-in pagination (e.g. for DynamoDB table scans and queries).  Always get everything you asked for.
+- One-line S3 file write, read, and delete.
 
-- Can automatically set SQS timeouts based on runtime data (can also be user-specified)
+- DynamoDB secondary indexes.
 
-- Supports moto mock and localstack. Handy for testing and CI.
+- Built-in pagination (e.g. for DynamoDB table scans and queries).  Always get everything you asked for.
 
+- Can automatically set SQS timeouts based on runtime data (can also be user-specified).
 
 ## Usage
 

awsimple-3.9.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+awsimple/__init__.py,sha256=yo9COyuJZonzuW_5h2yvsQOtTqnwh8e_iMsMBqQxir0,964
+awsimple/__version__.py,sha256=j9qPCE9XWQi2f7AWe1m9-8MYilnfMgESTbkSf6OE2LE,323
+awsimple/aws.py,sha256=NbRu1v_J5j2-pcefNZ5Xggii3mM9nHpeHMz9L9K9r-U,7653
+awsimple/cache.py,sha256=_LvPx76215t8KhAJOin6Pe2b4lWpB6kbKpdzgR4FeA4,7206
+awsimple/dynamodb.py,sha256=8OlGbMg7uU1rpWbkjK9HdHMSfopRBrLb3AFALGgfLHg,39156
+awsimple/dynamodb_miv.py,sha256=4xPxQDYkIM-BVDGyAre6uqwJHsxguEbHbof8ztt-V7g,4645
+awsimple/logs.py,sha256=s9FhdDFWjfxGCVDx-FNTPgJ-YN1AOAgz4HNxTVfRARE,4108
+awsimple/mock.py,sha256=eScbnxFF9xAosOAsL-NZgp_P-fezB6StQMkb85Y3TNo,574
+awsimple/platform.py,sha256=TObvLIVgRGh-Mh4ZCxFfxAmrn8KNMHktr6XkDZX8JYE,376
+awsimple/pubsub.py,sha256=HzwkjIwFicoV--30ctov8oVlKfXWe7eaLOF1dieV7qk,7692
+awsimple/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+awsimple/s3.py,sha256=ydYLDj51b38lYI6LsJlc4i0PDj2vnWBuSkzdRKcWAUg,23858
+awsimple/sns.py,sha256=T_FyN8eSmBPo213HOfB3UBlMrvtBK768IaRo_ks-7do,3526
+awsimple/sqs.py,sha256=9ZY7161CpmYpcxlCFIfW8bvMn9SGl4cgGR79I4MFLDk,17281
+awsimple-3.9.1.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
+awsimple-3.9.1.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
+awsimple-3.9.1.dist-info/METADATA,sha256=KL-CjrA-YV-N3eY0YypcWlsbEQX3-kH5I6zMbL8swmY,6640
+awsimple-3.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+awsimple-3.9.1.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
+awsimple-3.9.1.dist-info/RECORD,,

{awsimple-3.6.1.dist-info → awsimple-3.9.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.4.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

awsimple-3.6.1.dist-info/RECORD

@@ -1,18 +0,0 @@
-awsimple/__init__.py,sha256=8aFfqWFAvRPweoZkKncvHAW2ytTW_5-AJ0nnmYqgUBw,916
-awsimple/__version__.py,sha256=jGiJ18Y9OeSn8YGis7tf4vYZ6ctQjR1W3uTiRheczhc,323
-awsimple/aws.py,sha256=NbRu1v_J5j2-pcefNZ5Xggii3mM9nHpeHMz9L9K9r-U,7653
-awsimple/cache.py,sha256=_LvPx76215t8KhAJOin6Pe2b4lWpB6kbKpdzgR4FeA4,7206
-awsimple/dynamodb.py,sha256=8OlGbMg7uU1rpWbkjK9HdHMSfopRBrLb3AFALGgfLHg,39156
-awsimple/dynamodb_miv.py,sha256=4xPxQDYkIM-BVDGyAre6uqwJHsxguEbHbof8ztt-V7g,4645
-awsimple/logs.py,sha256=A2RmTT90pfFTthfENd7GSsEHSIBJXO8ICHPdA7sEsHY,4278
-awsimple/mock.py,sha256=eScbnxFF9xAosOAsL-NZgp_P-fezB6StQMkb85Y3TNo,574
-awsimple/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-awsimple/s3.py,sha256=ydYLDj51b38lYI6LsJlc4i0PDj2vnWBuSkzdRKcWAUg,23858
-awsimple/sns.py,sha256=dOx3VUS04xxeG1krGudN4A5fqoIpXeHqXNkBvfbr_6Q,3292
-awsimple/sqs.py,sha256=ejV9twP15X8-mZ9IHGEUlYWqufEcasYuPf1xlGQt2a8,15506
-awsimple-3.6.1.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
-awsimple-3.6.1.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
-awsimple-3.6.1.dist-info/METADATA,sha256=_pQy4RCP8b5GS1iuz3o-wHx_IQWxq0szAlXSrp40aNE,6377
-awsimple-3.6.1.dist-info/WHEEL,sha256=DnLRTWE75wApRYVsjgc6wsVswC54sMSJhAEd4xhDpBk,91
-awsimple-3.6.1.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
-awsimple-3.6.1.dist-info/RECORD,,