awsimple 5.0.0__py3-none-any.whl → 7.0.0__py3-none-any.whl

awsimple/__init__.py

@@ -9,5 +9,5 @@ from .dynamodb_miv import DynamoDBMIVUI, miv_string, get_time_us, miv_us_to_time
 from .s3 import S3Access, S3DownloadStatus, S3ObjectMetadata, BucketNotFound
 from .sqs import SQSAccess, SQSPollAccess, aws_sqs_long_poll_max_wait_time, aws_sqs_max_messages, get_all_sqs_queues
 from .sns import SNSAccess
-from .pubsub import PubSub
+from .pubsub import Pub, Sub
 from .logs import LogsAccess

awsimple/__version__.py

@@ -1,7 +1,7 @@
 __application_name__ = "awsimple"
 __title__ = __application_name__
 __author__ = "abel"
-__version__ = "5.0.0"
+__version__ = "7.0.0"
 __author_email__ = "j@abel.co"
 __url__ = "https://github.com/jamesabel/awsimple"
 __download_url__ = "https://github.com/jamesabel/awsimple"

awsimple/dynamodb.py

@@ -519,6 +519,7 @@ class DynamoDBAccess(CacheAccess):
             except ClientError as e:
                 log.warning(e)
         if self.metadata_table is not None:
+            self.metadata_table.create_metadata_table()
             self.metadata_table.update_table_mtime()
 
         return created
@@ -957,6 +958,8 @@ class _DynamoDBMetadataTable(DynamoDBAccess):
         self.mtime_f_string = "mtime_f"
         self.mtime_human_string = "mtime_human"
         super().__init__(metadata_table_name, **kwargs)
+
+    def create_metadata_table(self):
         self.create_table(partition_key=self.primary_partition_key, sort_key=self.primary_sort_key, partition_key_type=str, sort_key_type=str)
 
     def update_table_mtime(self):

awsimple/pubsub.py

@@ -6,8 +6,8 @@ import time
 from functools import lru_cache
 from typing import Any, Dict, List, Callable, Union
 from datetime import timedelta
-from multiprocessing import Process, Queue, Event
-from threading import Thread
+from threading import Thread, Event
+from queue import Queue
 from queue import Empty
 from logging import Logger
 import json
@@ -26,7 +26,9 @@ log = Logger(__application_name__)
 
 queue_timeout = timedelta(days=30).total_seconds()
 
-sqs_name = "sqs"
+SQS_NAME = "sqs"
+
+AWS_RESOURCE_PREFIX = "ps"  # for pubsub
 
 
 @typechecked()
@@ -42,7 +44,7 @@ def remove_old_queues(
         return removed
     for sqs_queue_name in get_all_sqs_queues(channel):
         sqs_metadata = _DynamoDBMetadataTable(
-            sqs_name, sqs_queue_name, profile_name=profile_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name
+            SQS_NAME, sqs_queue_name, profile_name=profile_name, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=region_name
         )
         mtime = sqs_metadata.get_table_mtime_f()
         if mtime is not None and time.time() - mtime > queue_timeout:
@@ -101,30 +103,33 @@ def _connect_sns_to_sqs(sqs: SQSPollAccess, sns: SNSAccess) -> None:
 
 class _SubscriptionThread(Thread):
     """
-    Thread to poll SQS for new messages and put them in a queue for the parent process to read.
+    Thread to poll SQS for new messages and put them in a queue for the parent thread to read.
     """
 
     @typechecked()
     def __init__(self, sqs: SQSPollAccess, new_event) -> None:
         super().__init__()
         self._sqs = sqs
-        self.new_event = new_event
         self.sub_queue = Queue()  # type: Queue[str]
+        self._exit_event = Event()
+        self._new_event = new_event
 
     def run(self):
-        # exit by terminating the parent process
-        while True:
+        while not self._exit_event.is_set():
             messages = self._sqs.receive_messages()  # long poll
             for message in messages:
                 message = json.loads(message.message)
                 self.sub_queue.put(message["Message"])
-                self.new_event.set()  # notify parent process that a new message is available
+                self._new_event.set()
+
+    def request_exit(self):
+        self._exit_event.set()
 
 
 @lru_cache
 def make_name_aws_safe(*args: str) -> str:
     """
-    Make a name safe for an SQS queue to subscribe to an SNS topic. AWS has a bunch of undocumented restrictions on names, so we just hash the name to a base36 string.
+    Make a name safe for an SQS queue to subscribe to an SNS topic. This ensures we adhere to name restrictions such as acceptable characters and length.
 
     :params: input name(s)
     :return: AWS safe name
@@ -135,18 +140,19 @@ def make_name_aws_safe(*args: str) -> str:
     return base36
 
 
-class PubSub(Process):
+class _PubSub(Thread):
 
     @typechecked()
     def __init__(
         self,
         channel: str,
-        node_name: str = get_node_name(),
-        sub_callback: Callable | None = None,
-        profile_name: Union[str, None] = None,
-        aws_access_key_id: Union[str, None] = None,
-        aws_secret_access_key: Union[str, None] = None,
-        region_name: Union[str, None] = None,
+        node_name: str,
+        sub_callback: Callable | None,
+        use_sub_queue: bool,
+        profile_name: Union[str, None],
+        aws_access_key_id: Union[str, None],
+        aws_secret_access_key: Union[str, None],
+        region_name: Union[str, None],
     ) -> None:
         """
         Pub and Sub.
@@ -155,12 +161,13 @@ class PubSub(Process):
         :param channel: Channel name (used for SNS topic name). This must not be a prefix of other channel names to avoid collisions (don't name one channel "a" and another "ab").
         :param node_name: Node name (SQS queue name suffix). Defaults to a combination of computer name and username, but can be passed in for customization and/or testing.
         :param sub_callback: Optional thread and process safe callback function to be called when a new message is received. The function should accept a single argument, which will be the message as a dictionary.
+        :param use_sub_queue: If True, use an internal queue to store received messages. If False, messages must be handled by the callback function. Default is False.
         """
-        self.aws_resource_prefix = "ps"  # for pubsub
-        self.channel = self.aws_resource_prefix + make_name_aws_safe(channel)  # prefix with ps (pubsub) to avoid collisions with other uses of SNS topics and SQS queues
+        self.channel = AWS_RESOURCE_PREFIX + make_name_aws_safe(channel)  # prefix with ps (pubsub) to avoid collisions with other uses of SNS topics and SQS queues
         self.node_name = node_name
-        self.sqs_queue_name = self.aws_resource_prefix + make_name_aws_safe(self.channel, self.node_name)
+        self.sqs_queue_name = AWS_RESOURCE_PREFIX + make_name_aws_safe(self.channel, self.node_name)
         self.sub_callback = sub_callback
+        self.use_sub_queue = use_sub_queue
 
         self.profile_name = profile_name
         self.aws_access_key_id = aws_access_key_id
@@ -170,9 +177,11 @@ class PubSub(Process):
         self._pub_queue = Queue()  # type: Queue[Dict[str, Any]]
         self._sub_queue = Queue()  # type: Queue[str]
 
-        self._new_event = Event()  # pub or sub sets this when a new message is available or has been sent
+        self._exit_event = Event()  # set this to request exit
+        self._new_event = Event()
+        self._new_event_wait_time = 10  # seconds
 
-        super().__init__()
+        super().__init__(daemon=True)  # make daemon so an instance of this thread exits when the main program exits
 
     def run(self):
 
@@ -184,8 +193,10 @@ class PubSub(Process):
             aws_secret_access_key=self.aws_secret_access_key,
             region_name=self.region_name,
         )
+        sns.create_topic()
+
         sqs_metadata = _DynamoDBMetadataTable(
-            sqs_name,
+            SQS_NAME,
             self.sqs_queue_name,
             profile_name=self.profile_name,
             aws_access_key_id=self.aws_access_key_id,
@@ -210,10 +221,14 @@ class PubSub(Process):
         sqs_metadata.update_table_mtime()  # update SQS use time (the existing infrastructure calls it a "table", but we're using it for the SQS queue)
         remove_old_queues(self.channel)  # clean up old queues
 
-        sqs_thread = _SubscriptionThread(sqs, self._new_event)
-        sqs_thread.start()
+        if self.sub_callback is None and not self.use_sub_queue:
+            # not being used as a subscriber
+            sqs_thread = None
+        else:
+            sqs_thread = _SubscriptionThread(sqs, self._new_event)
+            sqs_thread.start()
 
-        while True:
+        while not self._exit_event.is_set():
 
             # pub
             try:
@@ -224,21 +239,26 @@ class PubSub(Process):
                 pass
 
             # sub
-            try:
-                message_string = sqs_thread.sub_queue.get(False)
-                # if a callback is provided, call it, otherwise put the message in the sub queue for later retrieval
-                if self.sub_callback is None:
-                    self._sub_queue.put(message_string)
-                else:
-                    message = json.loads(message_string)
-                    self.sub_callback(message)
-                sqs_metadata.update_table_mtime()
-            except Empty:
-                pass  # no message
-
-            # this helps responsiveness
-            self._new_event.clear()
-            self._new_event.wait(3)  # race conditions can occur, so don't make this wait timeout too long
+            if sqs_thread is not None:
+                try:
+                    message_string = sqs_thread.sub_queue.get(False)
+                    if self.use_sub_queue:
+                        self._sub_queue.put(message_string)
+                    if self.sub_callback is not None:
+                        message = json.loads(message_string)
+                        self.sub_callback(message)
+                    sqs_metadata.update_table_mtime()
+                except Empty:
+                    pass  # no message
+
+            if self._new_event.wait(self._new_event_wait_time):  # timeout in case the new event technique fails
+                self._new_event.clear()
+
+        if sqs_thread is not None:
+            sqs_thread.request_exit()
+            sqs_thread.join(30)
+            if sqs_thread.is_alive():
+                log.error("sqs_thread did not exit cleanly")
 
     @typechecked()
     def publish(self, message: dict) -> None:
@@ -253,10 +273,12 @@ class PubSub(Process):
     @typechecked()
     def get_messages(self) -> List[Dict[str, Any]]:
         """
-        Get all available messages.
+        Get all available messages. Muse set sub_poll=True when creating the PubSub object to use this function.
 
         :return: list of messages as dictionaries
         """
+        if not self.use_sub_queue:
+            raise RuntimeError("use_sub_queue must be True to use get_messages()")
         messages = []
         while True:
             try:
@@ -267,3 +289,74 @@ class PubSub(Process):
             except Empty:
                 break
         return messages
+
+    def request_exit(self) -> None:
+        """
+        Request the process to exit.
+        """
+        self._exit_event.set()
+        self._new_event.set()
+
+
+class Pub(_PubSub):
+
+    @typechecked()
+    def __init__(
+        self,
+        channel: str,
+        node_name: str = get_node_name(),
+        profile_name: Union[str, None] = None,
+        aws_access_key_id: Union[str, None] = None,
+        aws_secret_access_key: Union[str, None] = None,
+        region_name: Union[str, None] = None,
+    ) -> None:
+        """
+        Pub only.
+        Create in a separate process to offload from main thread. Also facilitates use of moto mock in tests.
+
+        :param channel: Channel name (used for SNS topic name). This must not be a prefix of other channel names to avoid collisions (don't name one channel "a" and another "ab").
+        """
+        super().__init__(
+            channel=channel,
+            node_name=node_name,
+            sub_callback=None,  # pub only
+            use_sub_queue=False,  # pub only
+            profile_name=profile_name,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            region_name=region_name,
+        )
+
+
+class Sub(_PubSub):
+
+    @typechecked()
+    def __init__(
+        self,
+        channel: str,
+        node_name: str = get_node_name(),
+        sub_callback: Callable | None = None,
+        profile_name: Union[str, None] = None,
+        aws_access_key_id: Union[str, None] = None,
+        aws_secret_access_key: Union[str, None] = None,
+        region_name: Union[str, None] = None,
+    ) -> None:
+        """
+        Sub only.
+        Create in a separate process to offload from main thread. Also facilitates use of moto mock in tests.
+
+        :param channel: Channel name (used for SNS topic name). This must not be a prefix of other channel names to avoid collisions (don't name one channel "a" and another "ab").
+        :param node_name: Node name (SQS queue name suffix). Defaults to a combination of computer name and username, but can be passed in for customization and/or testing.
+        :param sub_callback: Optional callback function to be called when a new message is received. The function should accept a single argument, which will be the message as a dictionary.
+                             If this is not used, then get_messages() should be used to retrieve messages.
+        """
+        super().__init__(
+            channel=channel,
+            node_name=node_name,
+            sub_callback=sub_callback,
+            use_sub_queue=sub_callback is None,  # if no callback, use internal queue
+            profile_name=profile_name,
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+            region_name=region_name,
+        )

awsimple/sqs.py

@@ -230,9 +230,9 @@ class SQSAccess(AWSAccess):
                 if (queue := self._get_queue()) is None:
                     log.warning(f"could not get queue {self.queue_name}")
                 else:
-                    aws_messages = queue.receive_messages(
-                        MaxNumberOfMessages=min(max_number_of_messages, aws_sqs_max_messages), VisibilityTimeout=self.calculate_visibility_timeout(), WaitTimeSeconds=call_wait_time
-                    )
+                    visibility_timeout = self.calculate_visibility_timeout()
+                    max_number_of_messages = min(max_number_of_messages, aws_sqs_max_messages)  # AWS limit
+                    aws_messages = queue.receive_messages(MaxNumberOfMessages=max_number_of_messages, VisibilityTimeout=visibility_timeout, WaitTimeSeconds=call_wait_time)
 
                     for m in aws_messages:
                         if self.immediate_delete:

{awsimple-5.0.0.dist-info → awsimple-7.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: awsimple
-Version: 5.0.0
+Version: 7.0.0
 Summary: Simple AWS API for S3, DynamoDB, SNS, and SQS
 Home-page: https://github.com/jamesabel/awsimple
 Download-URL: https://github.com/jamesabel/awsimple

{awsimple-5.0.0.dist-info → awsimple-7.0.0.dist-info}/RECORD

@@ -1,21 +1,21 @@
-awsimple/__init__.py,sha256=RT9hATlcKnnvJYnSh7tCWEeeT2LyleL_vT1Zdz0nSkM,1051
-awsimple/__version__.py,sha256=xCHPAAnvsCqjRKpsIAQOEFC8V53loQAyz0sN0R7SAo0,323
+awsimple/__init__.py,sha256=1uECRpVL5WHv1HPyn9sKQpEMJHwlbb7vhkEv7cuS5C4,1053
+awsimple/__version__.py,sha256=IwHeKF_Imptby8oSPG0-2P79ZV5YxNB8SS8Sg5-vFgY,323
 awsimple/aws.py,sha256=NbRu1v_J5j2-pcefNZ5Xggii3mM9nHpeHMz9L9K9r-U,7653
 awsimple/cache.py,sha256=_LvPx76215t8KhAJOin6Pe2b4lWpB6kbKpdzgR4FeA4,7206
-awsimple/dynamodb.py,sha256=7MNxAutOCMTS4JSX-DLOwzaImJ2TzIc7kfQzQPAy5y8,41193
+awsimple/dynamodb.py,sha256=3xqqs31RvW_UAVOk6aY2QhlXcnlz42POFmOjrkokTK4,41290
 awsimple/dynamodb_miv.py,sha256=4xPxQDYkIM-BVDGyAre6uqwJHsxguEbHbof8ztt-V7g,4645
 awsimple/exceptions.py,sha256=Ew-S8YkHVWrZFI_Yik5n0cJ7Ss4Kig5JsEPQ-9z18SU,922
 awsimple/logs.py,sha256=s9FhdDFWjfxGCVDx-FNTPgJ-YN1AOAgz4HNxTVfRARE,4108
 awsimple/mock.py,sha256=eScbnxFF9xAosOAsL-NZgp_P-fezB6StQMkb85Y3TNo,574
 awsimple/platform.py,sha256=TObvLIVgRGh-Mh4ZCxFfxAmrn8KNMHktr6XkDZX8JYE,376
-awsimple/pubsub.py,sha256=Fxo3i4zo3GfBVhmGMGI4CQsRQ18x4Pp_bjpxSL-9xJY,10240
+awsimple/pubsub.py,sha256=lsGN9HUQDh4pSVjVB0Y4T5QoiI69kIy8-m0G27PUsWU,14000
 awsimple/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 awsimple/s3.py,sha256=OhWF1uv4oLmBF5jAhKIi918iUQxx4CX8bTEvQ5wLYr8,24050
 awsimple/sns.py,sha256=T_FyN8eSmBPo213HOfB3UBlMrvtBK768IaRo_ks-7do,3526
-awsimple/sqs.py,sha256=9ZY7161CpmYpcxlCFIfW8bvMn9SGl4cgGR79I4MFLDk,17281
-awsimple-5.0.0.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
-awsimple-5.0.0.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
-awsimple-5.0.0.dist-info/METADATA,sha256=98gj_2M_A4xwO4-y5c8LLsRML1pfnoKeqE2-LCyEvMU,6660
-awsimple-5.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-awsimple-5.0.0.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
-awsimple-5.0.0.dist-info/RECORD,,
+awsimple/sqs.py,sha256=nS_rD38gJObYeoXkDSg8Dv5jZTlxc79VRHFbVfmseis,17376
+awsimple-7.0.0.dist-info/licenses/LICENSE,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
+awsimple-7.0.0.dist-info/licenses/LICENSE.txt,sha256=d956YAgtDaxgxQmccyUk__EfhORZyBjvmJ8pq6zYTxk,1093
+awsimple-7.0.0.dist-info/METADATA,sha256=QdkdmQ4c0u0zQtsprenJgqyUcGcdfFEQ6suMSKSw1KI,6660
+awsimple-7.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+awsimple-7.0.0.dist-info/top_level.txt,sha256=mwqCoH_8vAaK6iYioiRbasXmVCQM7AK3grNWOKp2VHE,9
+awsimple-7.0.0.dist-info/RECORD,,