omnata-plugin-runtime 0.4.6a102__tar.gz → 0.4.7__tar.gz

{omnata_plugin_runtime-0.4.6a102 → omnata_plugin_runtime-0.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.6a102
+Version: 0.4.7
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.6a102 → omnata_plugin_runtime-0.4.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.4.6-a102"
+version = "0.4.7"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.4.6a102 → omnata_plugin_runtime-0.4.7}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -706,7 +706,7 @@ class OutboundSyncRequest(SyncRequest):
         logger.info("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+            for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
                 with attempt:
                     success, nchunks, nrows, _ = write_pandas(
                         conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
@@ -1345,7 +1345,7 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
                     with attempt:
                         logger.info(
                             f"Applying {len(results_df)} results to {self._full_results_table_name}"
@@ -1392,7 +1392,7 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*is being committed.*")):
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
                     with attempt:
                         logger.info(
                             f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
@@ -2097,50 +2097,61 @@ def managed_inbound_processing(concurrency: int):
 
             tasks:List[threading.Thread] = []
             logger.info(f"Creating {concurrency_to_use} worker(s) for retrieving records")
-
-            for i in range(concurrency_to_use):
-                # the dataframe/generator was put on the queue, so we remove it from the method args
-                task = threading.Thread(
-                    target=__managed_inbound_processing_worker,
-                    name=f"managed_inbound_processing_worker_{i}",
-                    args=(
-                        self,
-                        method,
-                        i,
-                        streams_queue,
-                        self._sync_request._thread_cancellation_token,
-                        method_args,
-                        method_kwargs,
-                    ),
+            # if concurrency is set to 1, we don't need to use threads at all
+            if concurrency_to_use == 1:
+                __managed_inbound_processing_worker(
+                    self,
+                    method,
+                    0,
+                    streams_queue,
+                    self._sync_request._thread_cancellation_token,
+                    method_args,
+                    method_kwargs,
                 )
-                tasks.append(task)
-                task.start()
-
-            # wait for workers to finish
-            while tasks:
-                for task in tasks[:]: # shallow copy so we can remove items from the list while iterating
-                    if not task.is_alive():
-                        task.join()  # Ensure the thread is fully finished
-                        tasks.remove(task)
-                        logger.info(f"Thread {task.name} has completed processing")
-                time.sleep(1)  # Avoid busy waiting
-            logger.info("All workers completed processing")
-
-            # it's possible that some records weren't applied, since they are processed asynchronously on a timer
-            #if self._sync_request.development_mode is False:
-            #    self._sync_request.apply_results_queue()
-            #self._sync_request._thread_cancellation_token.set()
-            ## the thread cancellation should be detected by the apply results tasks, so it finishes gracefully
-            #if (
-            #    self._sync_request.development_mode is False
-            #    and self._sync_request._apply_results_task is not None
-            #):
-            #    self._sync_request._apply_results_task.join()
-            if self._sync_request._thread_exception_thrown:
-                logger.info("Raising thread exception")
-                raise self._sync_request._thread_exception_thrown.exc_value
             else:
-                logger.info("No thread exception thrown")
+                for i in range(concurrency_to_use):
+                    # the dataframe/generator was put on the queue, so we remove it from the method args
+                    task = threading.Thread(
+                        target=__managed_inbound_processing_worker,
+                        name=f"managed_inbound_processing_worker_{i}",
+                        args=(
+                            self,
+                            method,
+                            i,
+                            streams_queue,
+                            self._sync_request._thread_cancellation_token,
+                            method_args,
+                            method_kwargs,
+                        ),
+                    )
+                    tasks.append(task)
+                    task.start()
+
+                # wait for workers to finish
+                while tasks:
+                    for task in tasks[:]: # shallow copy so we can remove items from the list while iterating
+                        if not task.is_alive():
+                            task.join()  # Ensure the thread is fully finished
+                            tasks.remove(task)
+                            logger.info(f"Thread {task.name} has completed processing")
+                    time.sleep(1)  # Avoid busy waiting
+                logger.info("All workers completed processing")
+
+                # it's possible that some records weren't applied, since they are processed asynchronously on a timer
+                #if self._sync_request.development_mode is False:
+                #    self._sync_request.apply_results_queue()
+                #self._sync_request._thread_cancellation_token.set()
+                ## the thread cancellation should be detected by the apply results tasks, so it finishes gracefully
+                #if (
+                #    self._sync_request.development_mode is False
+                #    and self._sync_request._apply_results_task is not None
+                #):
+                #    self._sync_request._apply_results_task.join()
+                if self._sync_request._thread_exception_thrown:
+                    logger.info("Raising thread exception")
+                    raise self._sync_request._thread_exception_thrown.exc_value
+                else:
+                    logger.info("No thread exception thrown")
             logger.info("Main managed_inbound_processing thread completing")
             return
 

{omnata_plugin_runtime-0.4.6a102 → omnata_plugin_runtime-0.4.7}/src/omnata_plugin_runtime/rate_limiting.py

@@ -354,10 +354,15 @@ class RetryWithLogging(Retry):
     """
         Adding extra logs before making a retry request     
     """
-    def __init__(self, *args: Any, thread_cancellation_token:threading.Event, **kwargs: Any) -> Any:
-        self.thread_cancellation_token = thread_cancellation_token
+    def __init__(self, *args: Any, **kwargs: Any) -> Any:
+        self.thread_cancellation_token:Optional[threading.Event] = None
         return super().__init__(*args, **kwargs)
     
+    def new(self, **kw):
+        new_retry = super().new(**kw)
+        new_retry.thread_cancellation_token = self.thread_cancellation_token
+        return new_retry
+    
     def sleep_for_retry(self, response=None):
         retry_after = self.get_retry_after(response)
         if retry_after:
@@ -406,13 +411,13 @@ class RateLimitedSession(requests.Session):
         self.statuses_to_include = statuses_to_include
 
         retry_strategy = RetryWithLogging(
-            thread_cancellation_token=thread_cancellation_token,
             total=max_retries,
             backoff_factor=backoff_factor,
             status_forcelist=statuses_to_include,
             allowed_methods=["HEAD", "GET", "OPTIONS", "POST", "PUT", "DELETE"],
             respect_retry_after_header=respect_retry_after_header
         )
+        retry_strategy.thread_cancellation_token = thread_cancellation_token
         adapter = HTTPAdapter(max_retries=retry_strategy)
         self.mount("https://", adapter)
         self.mount("http://", adapter)