omnata-plugin-runtime 0.3.20a63__py3-none-any.whl → 0.3.21a64__py3-none-any.whl

omnata_plugin_runtime/configuration.py

@@ -443,10 +443,9 @@ class StoredStreamConfiguration(SubscriptableBaseModel):
         None,
         description="The field to use as a cursor",
     )
-    # Composite primary keys is not really a thing in SaaS applications
-    primary_key_field: Optional[str] = Field(
+    primary_key_field: Optional[Union[str,List[str]]] = Field(
         None,
-        description="The field that will be used as primary key.",
+        description="The field(s) that will be used as primary key.",
     )
     storage_behaviour: InboundStorageBehaviour
     stream: StreamConfiguration
@@ -465,9 +464,17 @@ class StreamConfiguration(SubscriptableBaseModel):
         None,
         description="If true, the plugin controls the cursor field",
     )
-    source_defined_primary_key: Optional[str] = Field(
+    source_defined_primary_key: Optional[Union[str,List[str]]] = Field(
         None,
-        description="If true, the plugin controls the primary key",
+        description="If defined, the plugin controls the primary key field",
+    )
+    # For SaaS applications, typically the primary key is a single field, known to the plugin.
+    # For databases, the primary key can be composite, and the plugin may not know what it is if there is no key defined explicitly on the table.
+    # this flag was added to support this scenario. If it is to True, then source_defined_primary_key will be a list of fields.
+    # When this occurs, the plugin runtime will build the APP_IDENTIFIER from the fields in the order they are listed, by concatenating them.
+    primary_key_can_be_composite: Optional[bool] = Field(
+        False,
+        description="If true, the primary key can be composite",
     )
     default_cursor_field: Optional[str] = Field(
         None,

omnata_plugin_runtime/omnata_plugin.py

@@ -19,6 +19,7 @@ import http
 import json
 import queue
 import threading
+import time
 from abc import ABC, abstractmethod
 from decimal import Decimal
 from functools import partial, wraps, reduce
@@ -1200,7 +1201,8 @@ class InboundSyncRequest(SyncRequest):
                         raise ValueError(
                             f"Primary key field '{primary_key_field}' was not present in all records for stream {stream_name}"
                         )
-
+                # TODO: handle the scenario where the primary key field is a list, and concatenate the values to form the APP_IDENTIFIER
+                # Currently this is only implemented in the java runtime, since it's typically databases that use composite keys
                 results_df["APP_IDENTIFIER"] = results_df["RECORD_DATA"].apply(lambda x: get_nested_value(dict(x),primary_key_field))
             # ensure APP_IDENTIFIER is a string
             results_df["APP_IDENTIFIER"] = results_df["APP_IDENTIFIER"].apply(str)
@@ -1836,9 +1838,13 @@ def managed_outbound_processing(concurrency: int, batch_size: int):
                 task.start()
 
             # wait for workers to finish
-            for task in tasks:
-                task.join()
-                logger.info(f"Task {task.name} joined")
+            while tasks:
+                for task in tasks[:]: # shallow copy so we can remove items from the list while iterating
+                    if not task.is_alive():
+                        task.join()  # Ensure the thread is fully finished
+                        tasks.remove(task)
+                        logger.info(f"Thread {task.name} has completed processing")
+                time.sleep(1)  # Avoid busy waiting
             logger.info("All workers completed processing")
 
             # it's possible that some records weren't applied, since they are processed asynchronously on a timer
@@ -1892,11 +1898,14 @@ def __managed_inbound_processing_worker(
             logger.info(f"stream returned from queue: {stream}")
             # restore the first argument, was originally the dataframe/generator but now it's the appropriately sized dataframe
             try:
+                logger.info(f"worker {worker_index} processing stream {stream.stream_name}, invoking plugin class method {method.__name__}")
                 method(plugin_class_obj, *(stream, *method_args), **method_kwargs)
+                logger.info(f"worker {worker_index} completed processing stream {stream.stream_name}, marking as complete")
                 plugin_class_obj._sync_request.mark_stream_complete(stream.stream_name)
             except InterruptedWhileWaitingException:
                 # If an inbound run is cancelled while waiting for rate limiting, this should mean that
                 # the cancellation is handled elsewhere, so we don't need to do anything special here other than stop waiting
+                logger.info(f"worker {worker_index} interrupted while waiting, exiting")
                 return
             except Exception as e:
                 # logging this to the omnata_plugin logger in this way, 
@@ -1967,13 +1976,14 @@ def managed_inbound_processing(concurrency: int):
             for stream in streams_list:
                 streams_queue.put(stream)
 
-            tasks = []
+            tasks:List[threading.Thread] = []
             logger.info(f"Creating {concurrency_to_use} worker(s) for retrieving records")
 
             for i in range(concurrency_to_use):
                 # the dataframe/generator was put on the queue, so we remove it from the method args
                 task = threading.Thread(
                     target=__managed_inbound_processing_worker,
+                    name=f"managed_inbound_processing_worker_{i}",
                     args=(
                         self,
                         method,
@@ -1988,9 +1998,13 @@ def managed_inbound_processing(concurrency: int):
                 task.start()
 
             # wait for workers to finish
-            for task in tasks:
-                task.join()
-                logger.info("Task joined")
+            while tasks:
+                for task in tasks[:]: # shallow copy so we can remove items from the list while iterating
+                    if not task.is_alive():
+                        task.join()  # Ensure the thread is fully finished
+                        tasks.remove(task)
+                        logger.info(f"Thread {task.name} has completed processing")
+                time.sleep(1)  # Avoid busy waiting
             logger.info("All workers completed processing")
 
             # it's possible that some records weren't applied, since they are processed asynchronously on a timer

{omnata_plugin_runtime-0.3.20a63.dist-info → omnata_plugin_runtime-0.3.21a64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.3.20a63
+Version: 0.3.21a64
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.3.20a63.dist-info → omnata_plugin_runtime-0.3.21a64.dist-info}/RECORD

@@ -1,12 +1,12 @@
 omnata_plugin_runtime/__init__.py,sha256=w63LVME5nY-hQ4BBzfacy9kvTunwqHGs8iiSPGAX2ns,1214
 omnata_plugin_runtime/api.py,sha256=_N5ok5LN7GDO4J9n3yduXp3tpjmhpySY__U2baiygrs,6217
-omnata_plugin_runtime/configuration.py,sha256=AY_hHBd283VHx_rU9TzUol5VTzA6PHxxTltYWV7clCg,35113
+omnata_plugin_runtime/configuration.py,sha256=7cMekoY8CeZAJHpASU6tCMidF55Hzfr7CD74jtebqIY,35742
 omnata_plugin_runtime/forms.py,sha256=pw_aKVsXSz47EP8PFBI3VDwdSN5IjvZxp8JTjO1V130,18421
 omnata_plugin_runtime/logging.py,sha256=bn7eKoNWvtuyTk7RTwBS9UARMtqkiICtgMtzq3KA2V0,3272
-omnata_plugin_runtime/omnata_plugin.py,sha256=9t17Ug94v0IdXzUNx27v_U0itUVQV3k0Tqqx5b-djwE,99609
+omnata_plugin_runtime/omnata_plugin.py,sha256=ArEtB7LvbLBq9u39Y_2ivg24Hi1Xe-kcnzj5UG4mLP0,100971
 omnata_plugin_runtime/plugin_entrypoints.py,sha256=s-SrUnXaS6FaBq1igiJhcCB3Md_a6op-dp_g1H_55QU,27736
 omnata_plugin_runtime/rate_limiting.py,sha256=se6MftQI5NrVHaLb1hByPCgAESPQhkAgIG7KIU1clDU,16562
-omnata_plugin_runtime-0.3.20a63.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
-omnata_plugin_runtime-0.3.20a63.dist-info/METADATA,sha256=oCOEIy9kdmK2QKAfWS3MtmgRiL-rPEiVMj3a3aNzh-E,1604
-omnata_plugin_runtime-0.3.20a63.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-omnata_plugin_runtime-0.3.20a63.dist-info/RECORD,,
+omnata_plugin_runtime-0.3.21a64.dist-info/LICENSE,sha256=IMF9i4xIpgCADf0U-V1cuf9HBmqWQd3qtI3FSuyW4zE,26526
+omnata_plugin_runtime-0.3.21a64.dist-info/METADATA,sha256=AN68hzVHcChygxFH76s7iZtvROwge5NqHDNttfkRfko,1604
+omnata_plugin_runtime-0.3.21a64.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+omnata_plugin_runtime-0.3.21a64.dist-info/RECORD,,