omnata-plugin-runtime 0.4.7a104__tar.gz → 0.4.8a105__tar.gz

{omnata_plugin_runtime-0.4.7a104 → omnata_plugin_runtime-0.4.8a105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.4.7a104
+Version: 0.4.8a105
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.4.7a104 → omnata_plugin_runtime-0.4.8a105}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.4.7-a104"
+version = "0.4.8-a105"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.4.7a104 → omnata_plugin_runtime-0.4.8a105}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -1244,55 +1244,68 @@ class InboundSyncRequest(SyncRequest):
             )
             primary_key_field = None
             records = results_df.to_dict("records")
-            if (self._streams_dict[stream_name].stream.source_defined_primary_key is not None):
-                primary_key_field = self._streams_dict[stream_name].stream.source_defined_primary_key
+            # this extra bit of source_defined_primary_key logic is just to catch the situation where the plugin accidentally
+            # provides an empty list as the source defined primary key (SFMC plugin did this for a while, so some customers have [] here).
+            # This should be caught upstream during configuration, but we'll check here too.
+            # Note that source defined primary keys override any user choice.
+            primary_key_correctly_defined_by_source = False
+            if stream_obj.source_defined_primary_key is not None:
+                if isinstance(stream_obj.source_defined_primary_key,list):
+                    if len(stream_obj.source_defined_primary_key)>0:
+                        primary_key_correctly_defined_by_source = True
+                else:
+                    primary_key_correctly_defined_by_source = True
+            
+            if primary_key_correctly_defined_by_source:
+                primary_key_field = stream_obj.source_defined_primary_key
             elif self._streams_dict[stream_name].primary_key_field is not None:
                 primary_key_field = self._streams_dict[stream_name].primary_key_field
             else:
-                results_df["APP_IDENTIFIER"] = None
-            if primary_key_field is not None:
-                if isinstance(primary_key_field,list) and len(primary_key_field) == 1:
-                    # don't hash it if it's just a single value
-                    primary_key_field = primary_key_field[0]
-                if isinstance(primary_key_field,list):
-                    primary_key_fields = cast(List[str],primary_key_field)
-                    primary_key_fields = sorted(primary_key_fields)
-                    # handle the sitation where the primary key is a list of fields
-                    # first, check that all records contain all of the primary key fields
-                    if not all(
-                        all(
-                            field in record["RECORD_DATA"]
-                            for field in primary_key_fields
-                        )
-                        for record in records
-                    ):
-                        raise ValueError(
-                            f"Primary key fields '{primary_key_fields}' were not present in all records for stream {stream_name}"
-                        )
-                    # hash all of the primary key fields
-                    results_df["APP_IDENTIFIER"] = results_df["RECORD_DATA"].apply(lambda x: self.get_hash([str(x[field]) for field in primary_key_fields]))
-                else:
-                    # the primary key field could contain a nested field, so we need to check for that
-                    # we need to check that each record in the results contains the primary key field
-                    if not all(
-                        primary_key_field in record["RECORD_DATA"]
-                        for record in records
-                    ):
-                        if "." in primary_key_field:
-                            primary_key_field = primary_key_field.split(".")
-
-                            if not all(
-                                get_nested_value(record["RECORD_DATA"], primary_key_field)
-                                for record in records
-                            ):
-                                raise ValueError(
-                                    f"Primary key field '{primary_key_field}' was not present in all records for stream {stream_name}"
-                                )
-                        else:
+                # originally, we did not require primary keys for inbound syncs if they were doing the replace option
+                # when we brought in delete flagging, we began to mandate that primary keys are defined
+                raise ValueError(f"Stream {stream_name} does not have a primary key field defined")
+            if isinstance(primary_key_field,list) and len(primary_key_field) == 1:
+                # don't hash it if it's just a single value
+                primary_key_field = primary_key_field[0]
+            if isinstance(primary_key_field,list):
+                primary_key_fields = cast(List[str],primary_key_field)
+                primary_key_fields = sorted(primary_key_fields)
+                # handle the sitation where the primary key is a list of fields
+                # first, check that all records contain all of the primary key fields
+                if not all(
+                    all(
+                        field in record["RECORD_DATA"]
+                        for field in primary_key_fields
+                    )
+                    for record in records
+                ):
+                    raise ValueError(
+                        f"Primary key fields '{primary_key_fields}' were not present in all records for stream {stream_name}"
+                    )
+                # hash all of the primary key fields
+                results_df["APP_IDENTIFIER"] = results_df["RECORD_DATA"].apply(lambda x: self.get_hash([str(x[field]) for field in primary_key_fields]))
+            else:
+                # the primary key field could contain a nested field, so we need to check for that
+                # we need to check that each record in the results contains the primary key field
+                if not all(
+                    primary_key_field in record["RECORD_DATA"]
+                    for record in records
+                ):
+                    if "." in primary_key_field:
+                        primary_key_field = primary_key_field.split(".")
+
+                        if not all(
+                            get_nested_value(record["RECORD_DATA"], primary_key_field)
+                            for record in records
+                        ):
                             raise ValueError(
                                 f"Primary key field '{primary_key_field}' was not present in all records for stream {stream_name}"
                             )
-                    results_df["APP_IDENTIFIER"] = results_df["RECORD_DATA"].apply(lambda x: get_nested_value(dict(x),primary_key_field))
+                    else:
+                        raise ValueError(
+                            f"Primary key field '{primary_key_field}' was not present in all records for stream {stream_name}"
+                        )
+                results_df["APP_IDENTIFIER"] = results_df["RECORD_DATA"].apply(lambda x: get_nested_value(dict(x),primary_key_field))
             # ensure APP_IDENTIFIER is a string
             results_df["APP_IDENTIFIER"] = results_df["APP_IDENTIFIER"].apply(str)
             # the timestamps in Snowflake are TIMESTAMP_LTZ, so we upload in string format to ensure the