omnata-plugin-runtime 0.5.10a162__tar.gz → 0.6.0__tar.gz

{omnata_plugin_runtime-0.5.10a162 → omnata_plugin_runtime-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.5.10a162
+Version: 0.6.0
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.5.10a162 → omnata_plugin_runtime-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.5.10-a162"
+version = "0.6.0"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.5.10a162 → omnata_plugin_runtime-0.6.0}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -343,7 +343,7 @@ class SyncRequest(ABC):
         self._thread_exception_thrown = args
         logger.error("Thread exception", exc_info=True)
         self._thread_cancellation_token.set()  # this will tell the other threads to stop working
-        logger.info(
+        logger.debug(
             f"thread_cancellation_token: {self._thread_cancellation_token.is_set()}"
         )
 
@@ -384,7 +384,7 @@ class SyncRequest(ABC):
         Designed to be run in a thread, this method polls the results every 20 seconds and sends them back to Snowflake.
         """
         while not cancellation_token.is_set():
-            logger.info("apply results worker checking for results")
+            logger.debug("apply results worker checking for results")
             self.apply_results_queue()
             cancellation_token.wait(20)
         logger.info("apply results worker exiting")
@@ -411,7 +411,7 @@ class SyncRequest(ABC):
         workers, but that meant it's not being checked when the plugin doesn't use those decorators.
         """
         while not cancellation_token.is_set():
-            logger.info("cancel checking worker checking for cancellation")
+            logger.debug("cancel checking worker checking for cancellation")
             if (datetime.datetime.now(datetime.timezone.utc) > self._run_deadline):  # pylint: disable=protected-access
                 # if we've reached the deadline for the run, end it
                 self.deadline_reached = True
@@ -454,7 +454,7 @@ class SyncRequest(ABC):
                 api_limits_for_category = [x for x in self.api_limits if x.endpoint_category == endpoint_category]
                 if len(api_limits_for_category) > 0:
                     self.rate_limit_state_this_sync_and_branch[endpoint_category].prune_history(api_limits_for_category[0].request_rates)
-            logger.info(f"Updating rate limit state for sync {self._run_id}")
+            logger.debug(f"Updating rate limit state for sync {self._run_id}")
             update_rate_limit_result = self._plugin_message(
                 PluginMessageRateLimitState(rate_limit_state=self.rate_limit_state_this_sync_and_branch)
             )
@@ -466,7 +466,7 @@ class SyncRequest(ABC):
                 self.rate_limit_state_all = rate_limit_state_all
                 self.rate_limit_state_this_sync_and_branch = rate_limit_state_this_branch
         else:
-            logger.info("No rate limit state to update")
+            logger.debug("No rate limit state to update")
 
     @abstractmethod
     def apply_cancellation(self):
@@ -512,7 +512,7 @@ class SyncRequest(ABC):
             # if the rate limiting is going to require us to wait past the run deadline, we bail out now
             raise DeadlineReachedException()
         time_now = datetime.datetime.now(datetime.timezone.utc)
-        logger.info(
+        logger.debug(
             f"calculated wait until date was {wait_until}, comparing to {time_now}"
         )
 
@@ -542,7 +542,7 @@ class SyncRequest(ABC):
         Keep this to a very consise string, like 'Fetching records from API'.
         Avoid lengthy diagnostic messages, anything like this should be logged the normal way.
         """
-        logger.info(f"Activity update: {current_activity}")
+        logger.debug(f"Activity update: {current_activity}")
         return self._plugin_message(
             PluginMessageCurrentActivity(current_activity=current_activity)
         )
@@ -551,7 +551,7 @@ class SyncRequest(ABC):
         """
         Sends a message back to the plugin. This is used to send back the results of a sync run.
         """
-        logger.info(f"Sending plugin message: {message}")
+        logger.debug(f"Sending plugin message: {message}")
         with self._snowflake_query_lock:
             try:
                 # this is not ideal, but "Bind variable in stored procedure is not supported yet"
@@ -687,23 +687,22 @@ class OutboundSyncRequest(SyncRequest):
         """
         Merges all of the queued results and applies them
         """
-        logger.info("OutboundSyncRequest apply_results_queue")
+        logger.debug("OutboundSyncRequest apply_results_queue")
         if self._apply_results is not None:
             with self._apply_results_lock:
                 self._apply_results = [
                     x for x in self._apply_results if x is not None and len(x) > 0
                 ]  # remove any None/empty dataframes
                 if len(self._apply_results) > 0:
-                    logger.info(
+                    logger.debug(
                         f"Applying {len(self._apply_results)} batches of queued results"
                     )
                     # upload all cached apply results
                     all_dfs = pandas.concat(self._apply_results)
-                    #logger.info(f"applying: {all_dfs}")
                     self._apply_results_dataframe(all_dfs)
                     self._apply_results.clear()
                 else:
-                    logger.info("No queued results to apply")
+                    logger.debug("No queued results to apply")
 
     def apply_cancellation(self):
         """
@@ -729,7 +728,7 @@ class OutboundSyncRequest(SyncRequest):
         """
         Adds some results to the queue for applying asynchronously
         """
-        logger.info(f"Enqueueing {len(results)} results for upload")
+        logger.debug(f"Enqueueing {len(results)} results for upload")
         for required_column in ["IDENTIFIER", "RESULT", "SUCCESS"]:
             if required_column not in results.columns:
                 raise ValueError(
@@ -763,7 +762,7 @@ class OutboundSyncRequest(SyncRequest):
         results_df.set_index("IDENTIFIER", inplace=True, drop=False)
         results_df["APPLY_STATE_DATETIME"] = str(datetime.datetime.now().astimezone())
         if results_df is not None:
-            logger.info(
+            logger.debug(
                 f"Applying a queued results dataframe of {len(results_df)} records"
             )
             # change the success flag to an appropriate APPLY STATUS
@@ -803,7 +802,7 @@ class OutboundSyncRequest(SyncRequest):
         """
         Applies results for an outbound sync. This involves merging back onto the record state table
         """
-        logger.info("applying results to table")
+        logger.debug("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
             for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
@@ -819,7 +818,7 @@ class OutboundSyncRequest(SyncRequest):
                         raise ValueError(
                             f"Failed to write results to table {self._full_results_table_name}"
                         )
-                    logger.info(
+                    logger.debug(
                         f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                     )
 
@@ -831,12 +830,7 @@ class OutboundSyncRequest(SyncRequest):
         Parses the JSON in the transformed record column (Snowflake passes it as a string).
         Also when the mapper is a jinja template, renders it.
         """
-        # if data_frame is None:
-        #    logger.info(
-        #        "Dataframe wrapper skipping pre-processing as dataframe is None"
-        #    )
-        #    return None
-        logger.info(
+        logger.debug(
             f"Dataframe wrapper pre-processing {len(data_frame)} records"
         )
         if len(data_frame) > 0:
@@ -865,24 +859,24 @@ class OutboundSyncRequest(SyncRequest):
                 render_jinja
                 and "jinja_template" in data_frame.iloc[0]["TRANSFORMED_RECORD"]
             ):
-                logger.info("Rendering jinja template")
+                logger.debug("Rendering jinja template")
                 env = Environment()
                 # examine the plugin instance for jinja_filter decorated methods
                 if self.plugin_instance is not None:
                     for name in dir(self.plugin_instance):
                         member = getattr(self.plugin_instance, name)
                         if callable(member) and hasattr(member, "is_jinja_filter"):
-                            logger.info(f"Adding jinja filter to environment: {name}")
+                            logger.debug(f"Adding jinja filter to environment: {name}")
                             env.filters[name] = member
 
                 def do_jinja_render(jinja_env, row_value):
-                    logger.info(f"do_jinja_render: {row_value}")
+                    logger.debug(f"do_jinja_render: {row_value}")
                     jinja_template = jinja_env.from_string(row_value["jinja_template"])
                     try:
                         rendered_result = jinja_template.render(
                             {"row": row_value["source_record"]}
                         )
-                        logger.info(
+                        logger.debug(
                             f"Individual jinja rendering result: {rendered_result}"
                         )
                         return rendered_result
@@ -1043,7 +1037,7 @@ class InboundSyncRequest(SyncRequest):
         """
         Merges all of the queued results and applies them
         """
-        logger.info("InboundSyncRequest apply_results_queue ")
+        logger.debug("InboundSyncRequest apply_results_queue ")
         if self._apply_results is not None:
             with self._apply_results_lock:
                 results:List[pandas.DataFrame] = []
@@ -1061,12 +1055,11 @@ class InboundSyncRequest(SyncRequest):
                     results.extend(non_empty_dfs)  # remove any None/empty dataframes
                     stream_names.append(stream_name)
                 if len(results) > 0:
-                    logger.info(
+                    logger.debug(
                         f"Applying {len(results)} batches of queued results"
                     )
                     # upload all cached apply results
                     all_dfs = pandas.concat(results)
-                    #logger.info(f"applying: {all_dfs}")
                     self._apply_results_dataframe(stream_names, all_dfs)
                 # update the stream state object too
                 self._apply_latest_states()
@@ -1083,12 +1076,11 @@ class InboundSyncRequest(SyncRequest):
                         x for x in stream_results if x is not None and len(x) > 0
                     ])
                 if len(results) > 0:
-                    logger.info(
+                    logger.debug(
                         f"Applying {len(results)} batches of queued criteria deletes"
                     )
                     # upload all cached apply results
                     all_dfs = pandas.concat(results)
-                    #logger.info(f"applying: {all_dfs}")
                     self._apply_criteria_deletes_dataframe(all_dfs)
                     # clear the delete requests
                     self._apply_results_criteria_deletes = {}
@@ -1191,7 +1183,7 @@ class InboundSyncRequest(SyncRequest):
             # first, don't bother if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
                 if sum([x.memory_usage(index=True).sum() for x in all_dfs]) > 200000000:
-                    logger.info(f"Applying results queue immediately due to combined dataframe size")
+                    logger.debug(f"Applying results queue immediately due to combined dataframe size")
                     self.apply_results_queue()
     
     def delete_by_criteria(self, stream_name: str, criteria: Dict[str, Any]):
@@ -1215,7 +1207,7 @@ class InboundSyncRequest(SyncRequest):
         # STREAM_NAME: string
         # DELETE_CRITERIA: object
         with self._apply_results_lock:
-            logger.info(
+            logger.debug(
                 f"Enqueuing {len(criteria)} delete criteria for stream {stream_name} for upload"
             )
             existing_results: List[pandas.DataFrame] = []
@@ -1237,7 +1229,7 @@ class InboundSyncRequest(SyncRequest):
             # first, don't both if the count is less than 10000, since it's unlikely to be even close
             if combined_length > 10000:
                 if sum([x.memory_usage(index=True).sum() for x in all_dfs]) > 200000000:
-                    logger.info(f"Applying criteria deletes queue immediately due to combined dataframe size")
+                    logger.debug(f"Applying criteria deletes queue immediately due to combined dataframe size")
                     self.apply_results_queue() 
 
     def mark_stream_complete(self, stream_name: str):
@@ -1332,7 +1324,7 @@ class InboundSyncRequest(SyncRequest):
             raise ValueError(
                 f"Cannot preprocess results for stream {stream_name} as its configuration doesn't exist"
             )
-        logger.info(f"preprocessing for stream: {self._streams_dict[stream_name]}")
+        logger.debug(f"preprocessing for stream: {self._streams_dict[stream_name]}")
         if len(results) > 0:
             if isinstance(is_delete, list):
                 if len(results) != len(is_delete):
@@ -1469,7 +1461,7 @@ class InboundSyncRequest(SyncRequest):
             with self._snowflake_query_lock:
                 for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
                     with attempt:
-                        logger.info(
+                        logger.debug(
                             f"Applying {len(results_df)} results to {self._full_results_table_name}"
                         )
                         # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
@@ -1485,7 +1477,7 @@ class InboundSyncRequest(SyncRequest):
                             raise ValueError(
                                 f"Failed to write results to table {self._full_results_table_name}"
                             )
-                        logger.info(
+                        logger.debug(
                             f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                         )
                         # temp tables aren't allowed
@@ -1498,7 +1490,7 @@ class InboundSyncRequest(SyncRequest):
                         for stream_name in stream_names:
                             self._results_exist[stream_name] = True
         else:
-            logger.info("Results dataframe is empty, not applying")
+            logger.debug("Results dataframe is empty, not applying")
 
     def _apply_latest_states(self):
         """
@@ -1516,7 +1508,7 @@ class InboundSyncRequest(SyncRequest):
             with self._snowflake_query_lock:
                 for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
                     with attempt:
-                        logger.info(
+                        logger.debug(
                             f"Applying {len(results_df)} criteria deletes to {self._criteria_deletes_table_name}"
                         )
                         # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
@@ -1531,12 +1523,12 @@ class InboundSyncRequest(SyncRequest):
                             raise ValueError(
                                 f"Failed to write results to table {self._criteria_deletes_table_name}"
                             )
-                        logger.info(
+                        logger.debug(
                             f"Wrote {nrows} rows and {nchunks} chunks to table {self._criteria_deletes_table_name}"
                         )
                         return
         else:
-            logger.info("Results dataframe is empty, not applying")
+            logger.debug("Results dataframe is empty, not applying")
 
 
 class ConnectResponse(SubscriptableBaseModel):
@@ -1870,7 +1862,7 @@ class FixedSizeGenerator:
         self.generator = generator
         # handle dataframe as well as a dataframe generator, just to be more flexible
         if self.generator.__class__.__name__ == "DataFrame":
-            logger.info(
+            logger.debug(
                 f"Wrapping a dataframe of length {len(self.generator)} in a map so it acts as a generator"
             )
             self.generator = map(lambda x: x, [self.generator])
@@ -1880,14 +1872,14 @@ class FixedSizeGenerator:
 
     def __next__(self):
         with self.thread_lock:
-            logger.info(f"initial leftovers: {self.leftovers}")
+            logger.debug(f"initial leftovers: {self.leftovers}")
             records_df = self.leftovers
             self.leftovers = None
             try:
                 # build up a dataframe until we reach the batch size
                 while records_df is None or len(records_df) < self.batch_size:
                     current_count = 0 if records_df is None else len(records_df)
-                    logger.info(
+                    logger.debug(
                         f"fetching another dataframe from the generator, got {current_count} out of a desired {self.batch_size}"
                     )
                     next_df = next(self.generator)
@@ -1901,32 +1893,31 @@ class FixedSizeGenerator:
                             f"Dataframe generator provided an unexpected object, type {next_df.__class__.__name__}"
                         )
                     if next_df is None and records_df is None:
-                        logger.info(
+                        logger.debug(
                             "Original and next dataframes were None, returning None"
                         )
                         return None
                     records_df = pandas.concat([records_df, next_df])
-                    logger.info(
+                    logger.debug(
                         f"after concatenation, dataframe has {len(records_df)} records"
                     )
             except StopIteration:
-                logger.info("FixedSizeGenerator consumed the last pandas batch")
+                logger.debug("FixedSizeGenerator consumed the last pandas batch")
 
             if records_df is None:
-                logger.info("No records left, returning None")
+                logger.debug("No records left, returning None")
                 return None
             elif records_df is not None and len(records_df) > self.batch_size:
-                logger.info(
+                logger.debug(
                     f"putting {len(records_df[self.batch_size:])} records back ({len(records_df)} > {self.batch_size})"
                 )
                 self.leftovers = records_df[self.batch_size :].reset_index(drop=True)
                 records_df = records_df[0 : self.batch_size].reset_index(drop=True)
             else:
                 current_count = 0 if records_df is None else len(records_df)
-                logger.info(
+                logger.debug(
                     f"{current_count} records does not exceed batch size, not putting any back"
                 )
-            #logger.info(f"FixedSizeGenerator about to return dataframe {records_df}")
             return records_df
 
     def __iter__(self):
@@ -1948,7 +1939,7 @@ def __managed_outbound_processing_worker(
     Consumes a fixed sized set of records by passing them to the wrapped function,
     while adhering to the defined API constraints.
     """
-    logger.info(
+    logger.debug(
         f"worker {worker_index} processing. Cancelled: {cancellation_token.is_set()}"
     )
     while not cancellation_token.is_set():
@@ -1959,13 +1950,13 @@ def __managed_outbound_processing_worker(
         records_df = next(dataframe_generator)
         #logger.info(f"records returned from dataframe generator: {records_df}")
         if records_df is None:
-            logger.info(f"worker {worker_index} has no records left to process")
+            logger.debug(f"worker {worker_index} has no records left to process")
             return
         elif len(records_df) == 0:
-            logger.info(f"worker {worker_index} has 0 records left to process")
+            logger.debug(f"worker {worker_index} has 0 records left to process")
             return
 
-        logger.info(
+        logger.debug(
             f"worker {worker_index} fetched {len(records_df)} records for processing"
         )
         # threads block while waiting for their allocation of records, it's possible there's been
@@ -1975,7 +1966,7 @@ def __managed_outbound_processing_worker(
                 f"worker {worker_index} exiting before applying records, due to cancellation"
             )
             return
-        logger.info(f"worker {worker_index} processing {len(records_df)} records")
+        logger.debug(f"worker {worker_index} processing {len(records_df)} records")
         # restore the first argument, was originally the dataframe/generator but now it's the appropriately sized dataframe
         try:
             results_df = method(
@@ -1988,7 +1979,7 @@ def __managed_outbound_processing_worker(
                 f"worker {worker_index} interrupted while waiting for rate limiting, exiting"
             )
             return
-        logger.info(
+        logger.debug(
             f"worker {worker_index} received {len(results_df)} results, enqueueing"
         )
 
@@ -2000,7 +1991,7 @@ def __managed_outbound_processing_worker(
         outbound_sync_request.enqueue_results(
             results_df
         )  # pylint: disable=protected-access
-        logger.info(
+        logger.debug(
             f"worker {worker_index} enqueueing results"
         )
 
@@ -2058,7 +2049,7 @@ def managed_outbound_processing(concurrency: int, batch_size: int):
             # put the record iterator on the queue, ready for the first task to read it
             fixed_size_generator = FixedSizeGenerator(dataframe_arg, batch_size=batch_size)
             tasks:List[threading.Thread] = []
-            logger.info(f"Creating {concurrency} worker(s) for applying records")
+            logger.debug(f"Creating {concurrency} worker(s) for applying records")
             # just in case
             threading.excepthook = self._sync_request.thread_exception_hook
             for i in range(concurrency):
@@ -2132,17 +2123,17 @@ def __managed_inbound_processing_worker(
     """
     while not cancellation_token.is_set():
         # Get our generator object out of the queue
-        logger.info(
+        logger.debug(
             f"worker {worker_index} processing. Cancelled: {cancellation_token.is_set()}. Method args: {len(method_args)}. Method kwargs: {len(method_kwargs.keys())} ({','.join(method_kwargs.keys())})"
         )
         try:
             stream: StoredStreamConfiguration = streams_queue.get_nowait()
-            logger.info(f"stream returned from queue: {stream}")
+            logger.debug(f"stream returned from queue: {stream}")
             # restore the first argument, was originally the dataframe/generator but now it's the appropriately sized dataframe
             try:
-                logger.info(f"worker {worker_index} processing stream {stream.stream_name}, invoking plugin class method {method.__name__}")
+                logger.debug(f"worker {worker_index} processing stream {stream.stream_name}, invoking plugin class method {method.__name__}")
                 result = method(plugin_class_obj, *(stream, *method_args), **method_kwargs)
-                logger.info(f"worker {worker_index} completed processing stream {stream.stream_name}")
+                logger.debug(f"worker {worker_index} completed processing stream {stream.stream_name}")
                 if result is not None and result is False:
                     logger.info(f"worker {worker_index} requested that {stream.stream_name} be not marked as complete")
                 else:
@@ -2168,7 +2159,7 @@ def __managed_inbound_processing_worker(
                                 exc_info=True, 
                                 extra={'stream_name':stream.stream_name})
         except queue.Empty:
-            logger.info("streams queue is empty")
+            logger.debug("streams queue is empty")
             return
 
 
@@ -2196,7 +2187,7 @@ def managed_inbound_processing(concurrency: int):
                 raise ValueError(
                     "To use the managed_inbound_processing decorator, you must attach an apply request to the plugin instance (via the _sync_request property)"
                 )
-            logger.info(f"managed_inbound_processing invoked with {len(method_args)} positional arguments and {len(method_kwargs)} named arguments ({','.join(method_kwargs.keys())})")
+            logger.debug(f"managed_inbound_processing invoked with {len(method_args)} positional arguments and {len(method_kwargs)} named arguments ({','.join(method_kwargs.keys())})")
             if self._sync_request.development_mode is True:
                 concurrency_to_use = 1 # disable concurrency when running in development mode, it interferes with pyvcr
             else:

{omnata_plugin_runtime-0.5.10a162 → omnata_plugin_runtime-0.6.0}/src/omnata_plugin_runtime/plugin_entrypoints.py

@@ -322,7 +322,7 @@ class PluginEntrypoint:
         sync_parameters: Dict,
         selected_streams: Optional[List[str]], # None to return all streams without requiring schema
     ):
-        logger.info("Entered list_streams method")
+        logger.debug("Entered list_streams method")
         oauth_secret_name = normalise_nulls(oauth_secret_name)
         other_secrets_name = normalise_nulls(other_secrets_name)
         connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)

{omnata_plugin_runtime-0.5.10a162 → omnata_plugin_runtime-0.6.0}/src/omnata_plugin_runtime/rate_limiting.py

@@ -122,7 +122,7 @@ class ApiLimits(SubscriptableBaseModel):
         The resulting timestamp is when the next request can be made (if it's in the past, it can be done immediately)
         If multiple rate limits exist, the maximum timestamp is used (i.e. the most restrictive rate limit applies)
         """
-        logger.info(
+        logger.debug(
             f"calculating wait time, given previous requests as {rate_limit_state.previous_request_timestamps}"
         )
         if self.request_rates is None: