omnata-plugin-runtime 0.3.23a68__tar.gz → 0.3.24__tar.gz

{omnata_plugin_runtime-0.3.23a68 → omnata_plugin_runtime-0.3.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.3.23a68
+Version: 0.3.24
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.3.23a68 → omnata_plugin_runtime-0.3.24}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.3.23-a68"
+version = "0.3.24"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.3.23a68 → omnata_plugin_runtime-0.3.24}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -1261,7 +1261,7 @@ class InboundSyncRequest(SyncRequest):
             )
         ]
     
-    def get_hash(keys:List[str]) -> str:
+    def get_hash(self, keys:List[str]) -> str:
         """
         Creates a hash from a list of keys. 
         The function will join the keys with an underscore and then create a 
@@ -1823,30 +1823,40 @@ def managed_outbound_processing(concurrency: int, batch_size: int):
     def actual_decorator(method):
         @wraps(method)
         def _impl(self: OmnataPlugin, *method_args, **method_kwargs):
+            logger.info(f"managed_outbound_processing invoked with {len(method_args)} positional arguments and {len(method_kwargs)} named arguments ({','.join(method_kwargs.keys())})")
             if self._sync_request is None:  # pylint: disable=protected-access
                 raise ValueError(
                     "To use the managed_outbound_processing decorator, you must attach a sync request to the plugin instance (via the _sync_request property)"
                 )
-            # if self._sync_request.api_limits is None:
-            #    raise ValueError('To use the managed_outbound_processing decorator, API constraints must be defined. These can be provided in the response to the connect method')
             logger.info(f"Batch size: {batch_size}. Concurrency: {concurrency}")
-            if len(method_args) == 0:
-                raise ValueError(
-                    "You must provide at least one method argument, and the first argument must be a DataFrame or DataFrame generator (from outbound_sync_request.get_records)"
-                )
-            first_arg = method_args[0]
-            #logger.info(first_arg.__class__.__name__)
-            if first_arg.__class__.__name__ == "DataFrame":
-                logger.info("managed_outbound_processing received a DataFrame")
-            elif hasattr(first_arg, "__next__"):
-                logger.info("managed_outbound_processing received an iterator function")
-            else:
-                raise ValueError(
-                    f"The first argument to a @managed_outbound_processing method must be a DataFrame or DataFrame generator (from outbound_sync_request.get_records). Instead, a {first_arg.__class__.__name__} was provided."
-                )
+
+            dataframe_arg = None
+            if 'dataframe' in method_kwargs:
+                dataframe_arg = method_kwargs['dataframe']
+                del method_kwargs['dataframe']
+                if dataframe_arg.__class__.__name__ != "DataFrame":
+                    raise ValueError(
+                        f"The 'dataframe' named argument to the @managed_outbound_processing must be a DataFrame. Instead, a {dataframe_arg.__class__.__name__} was provided."
+                    )
+                
+            elif 'dataframe_generator' in method_kwargs:
+                dataframe_arg = method_kwargs['dataframe_generator']
+                del method_kwargs['dataframe_generator']
+                if not hasattr(dataframe_arg, "__next__"):
+                    raise ValueError(
+                        f"The 'dataframe_generator' named argument to the @managed_outbound_processing must be an iterator function. Instead, a {dataframe_arg.__class__.__name__} was provided."
+                    )
+            # if the dataframe was provided as the first argument, we'll use that
+            if dataframe_arg is None and len(method_args) > 0:
+                dataframe_arg = method_args[0]
+                if dataframe_arg.__class__.__name__ != "DataFrame" and not hasattr(dataframe_arg, "__next__"):
+                    raise ValueError(
+                        f"The first argument to a @managed_outbound_processing method must be a DataFrame or DataFrame generator (from outbound_sync_request.get_records). Instead, a {first_arg.__class__.__name__} was provided. Alternatively, you can provide these via the 'dataframe' or 'dataframe_generator' named arguments."
+                    )
+                method_args = method_args[1:]
 
             # put the record iterator on the queue, ready for the first task to read it
-            fixed_size_generator = FixedSizeGenerator(first_arg, batch_size=batch_size)
+            fixed_size_generator = FixedSizeGenerator(dataframe_arg, batch_size=batch_size)
             tasks:List[threading.Thread] = []
             logger.info(f"Creating {concurrency} worker(s) for applying records")
             # just in case
@@ -1862,7 +1872,7 @@ def managed_outbound_processing(concurrency: int, batch_size: int):
                         i,
                         fixed_size_generator,
                         self._sync_request._thread_cancellation_token,
-                        method_args[1:],
+                        method_args,
                         method_kwargs,
                     ),
                 )
@@ -1923,7 +1933,7 @@ def __managed_inbound_processing_worker(
     while not cancellation_token.is_set():
         # Get our generator object out of the queue
         logger.info(
-            f"worker {worker_index} processing. Cancelled: {cancellation_token.is_set()}"
+            f"worker {worker_index} processing. Cancelled: {cancellation_token.is_set()}. Method args: {len(method_args)}. Method kwargs: {len(method_kwargs.keys())} ({','.join(method_kwargs.keys())})"
         )
         try:
             stream: StoredStreamConfiguration = streams_queue.get_nowait()
@@ -1982,27 +1992,26 @@ def managed_inbound_processing(concurrency: int):
                 raise ValueError(
                     "To use the managed_inbound_processing decorator, you must attach an apply request to the plugin instance (via the _sync_request property)"
                 )
-            
+            logger.info(f"managed_inbound_processing invoked with {len(method_args)} positional arguments and {len(method_kwargs)} named arguments ({','.join(method_kwargs.keys())})")
             if self._sync_request.development_mode is True:
                 concurrency_to_use = 1 # disable concurrency when running in development mode, it interferes with pyvcr
             else:
                 concurrency_to_use = concurrency
-            # if self._sync_request.api_limits is None:
-            #    raise ValueError('To use the managed_inbound_processing decorator, API constraints must be defined. These can be provided in the response to the connect method')
-            if len(method_args) == 0:
-                raise ValueError(
-                    "You must provide at least one method argument, and the first argument must be a list of StoredStreamConfigurations (from inbound_sync_request.streams)"
-                )
-            first_arg: List[StoredStreamConfiguration] = method_args[0]
-            logger.info(first_arg.__class__.__name__)
-            if first_arg.__class__.__name__ == "list":
-                logger.info("managed_inbound_processing received a list")
-            else:
-                raise ValueError(
-                    f"The first argument to a @managed_inbound_processing method must be a list of StoredStreamConfigurations. Instead, a {first_arg.__class__.__name__} was provided."
-                )
+            stream_list_arg: List[StoredStreamConfiguration] = None
+            if 'streams' in method_kwargs:
+                stream_list_arg = cast(List[StoredStreamConfiguration],method_kwargs['streams'])
+                del method_kwargs['streams']
+            if stream_list_arg is None and len(method_args) > 0:
+                stream_list_arg = cast(List[StoredStreamConfiguration],method_args[0])
+                if stream_list_arg.__class__.__name__ != "list":
+                    raise ValueError(
+                        f"The first argument to a @managed_inbound_processing method must be a list of StoredStreamConfigurations if the 'streams' named argument is not provided. Instead, a {stream_list_arg.__class__.__name__} was provided."
+                    )
+                method_args = method_args[1:]
+            if stream_list_arg is None:
+                raise ValueError("You must provide a list of StoredStreamConfiguration objects to the method, either as the first argument or as a named argument 'streams'")
 
-            streams_list: List[StoredStreamConfiguration] = first_arg
+            streams_list = stream_list_arg
             # create a queue full of all the streams to process
             streams_queue = queue.Queue()
             for stream in streams_list:
@@ -2022,7 +2031,7 @@ def managed_inbound_processing(concurrency: int):
                         i,
                         streams_queue,
                         self._sync_request._thread_cancellation_token,
-                        method_args[1:],
+                        method_args,
                         method_kwargs,
                     ),
                 )