dc-python-sdk 1.5.28__tar.gz → 1.5.29__tar.gz

{dc_python_sdk-1.5.28/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.28
+Version: 1.5.29
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.28"
+version = "1.5.29"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.28
+version = 1.5.29
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.28
+Version: 1.5.29
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/src/dc_sdk/app.py

@@ -26,7 +26,7 @@ def run_pipeline():
             pipeline_run_history_id = api.create_new_history(PipelineEnvironment.pipeline_id)
             PipelineEnvironment.set_pipeline_run_history_id(pipeline_run_history_id)
         
-        pipeline_conductor = PipelineConductor(PipelineEnvironment.task, pipeline_id=PipelineEnvironment.pipeline_id, pipeline_run_history_id=PipelineEnvironment.pipeline_run_history_id)
+        pipeline_conductor = PipelineConductor(PipelineEnvironment.task, pipeline_id=PipelineEnvironment.pipeline_id, pipeline_run_history_id=PipelineEnvironment.pipeline_run_history_id, pipeline_mapping_id=PipelineEnvironment.pipeline_mapping_id if PipelineEnvironment.pipeline_mapping_id else None)
         pipeline_conductor.internal_log(pipeline_conductor.log_templates.INTERNAL_CONNECTOR_START.format(TASK.title(), pipeline_conductor.pipeline_details.connector_nm))
     except Exception as e:
         error_trace = traceback.format_exc()
@@ -65,6 +65,11 @@ def run_pipeline():
         if TASK == TasksEnum.SOURCE.value:
             pipeline_conductor.authenticate_source()
             pipeline_conductor.pipeline_details.increment_stage()
+            
+            if not pipeline_conductor.pipeline_details.pipeline_mapping_json:
+                pipeline_conductor.configure_fields()
+
+            pipeline_conductor.pipeline_details.increment_stage()
             pipeline_conductor.get_data()
             
             pipeline_conductor.pipeline_details.increment_stage()

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/src/dc_sdk/src/models/pipeline_details.py

@@ -11,6 +11,7 @@ class PipelineDetails:
         self.destination_object_id = row_data['destination_object_id']
         self.pipeline_mapping_json = row_data['pipeline_mapping_json']
         self.update_method_cd = row_data['update_method_cd']
+        self.primary_key_column_nm = row_data['primary_key_column_nm']
         self.source_connector_id = row_data['source_connector_id']
         self.source_connector_nm = row_data['source_connector_nm']
         self.destination_connector_id = row_data['destination_connector_id']

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/src/dc_sdk/src/pipeline.py

@@ -33,13 +33,14 @@ class PipelineConductor:
         self.prefix = kwargs.get("prefix")
         self.update_method = kwargs.get("update_method")
         self.mapping = kwargs.get("mapping")
+        self.pipeline_object_id = kwargs.get("pipeline_mapping_id")
         self.successful_keys = []
         self.config = PipelineEnvironment
         self.api = api or DataConnectorAPI()
         self.aws = aws or AwsService(PipelineEnvironment.aws_s3_bucket)
 
         # Set Pipeline Details
-        self.pipeline_details = self._get_pipeline_details() if mode == "prod" else self._get_pipeline_details_dev()
+        self.pipeline_details = self._get_pipeline_details()
 
         self.credentials = self._get_credentials() if mode == "prod" else kwargs.get('credentials')
         
@@ -202,10 +203,22 @@ class PipelineConductor:
             if unhandled:
                 print("An unrecognized issue has occurred on our side. Our team will be in contact within 24-48 hours, or try emailing support@dataconnector.com.")
 
-        
     def update_history(self, payload):
         self.api.put("{0}/history".format(self.pipeline_run_history_id), payload)
 
+    def configure_fields(self):
+        # get fields from connector
+        fields = self.connector.get_fields(self.pipeline_details.source_object_id)
+
+        if not fields:
+            raise errors.NoFieldsFoundError(self.pipeline_details.source_object_id)
+
+        mapping = [ { "column": field["field_id"], "mapped": field["field_id"] } for field in fields ]
+
+        self.pipeline_details.pipeline_mapping_json = json.dumps(mapping)
+
+        self.api.create_pipeline_mapping(self.pipeline_id, self.pipeline_details.pipeline_mapping_json)
+
     def _process_rows(self, rows, max_allowed=None):
         # Check if we have a limit and need to truncate rows
         limit_reached = False
@@ -269,27 +282,7 @@ class PipelineConductor:
         return self.aws.decrypt_customer_data_object(encyption_txt, self.pipeline_details.customer_metadata_uuid)
 
     def _get_pipeline_details(self):
-        return self.api.get_pipeline_details(str(self.pipeline_id), self.task, str(self.pipeline_run_history_id))
-        
-    def _get_pipeline_details_dev(self):
-        class PipelineDetail:
-            def __init__(self, object_id, filters, mapping, update_method) -> None:
-                self.source_object_id = object_id
-                self.destination_object_id = object_id
-                self.pipeline_mapping_json = mapping
-                self.update_method_cd = update_method
-                self.source_credential_nm = "Test"
-                self.destination_credential_nm = "Test"
-                # TODO: UPDATE OPTIONS
-                self.options = None
-                # self.filtered_column_nm = filters['filtered_column_nm'] if 'filtered_'
-                # self.start_selection_nm = filters['start_selection_nm']
-                # self.start_value_txt =filters['start_value_txt']
-                # self.end_selection_nm = filters['end_selection_nm']
-                # self.end_value_txt = filters['end_value_txt']
-                # self.timezone_offset_nbr = filters['timezone_offset_nbr']
-
-        return PipelineDetail(self.object_id, self.filters, self.mapping, self.update_method)
+        return self.api.get_pipeline_details(str(self.pipeline_id), self.task, str(self.pipeline_run_histoy_id), pipeline_mapping_id=self.pipeline_mapping_id)
 
     def _get_batch_row_count(self):
         results = self.connector.get_data(

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/src/dc_sdk/src/services/api.py

@@ -45,13 +45,16 @@ class DataConnectorAPI:
 
         self.post('logv2', payload)
 
-    def get_pipeline_details(self, pipeline_id: str, task, pipeline_run_history_id: str):
-        json = self.get(pipeline_id)
+    def get_pipeline_details(self, pipeline_id: str, task, pipeline_run_history_id: str, pipeline_mapping_id: str = None):
+        url = f"{pipeline_id}"
+        if pipeline_object_id:
+            url = f"{url}?PipelineMappingID={pipeline_mapping_id}"
+        json = self.get(url)
 
         return PipelineDetails(json, task, pipeline_id, pipeline_run_history_id)
 
     def create_new_history(self, pipeline_id):
-        response = self.post(f"{pipeline_id}/history", None)
+        response = self.post(f"{pipeline_id}/history", {"PipelineMappingID": self.pipeline_mapping_id})
 
         return response['PipelineRunHistoryID']
 

{dc_python_sdk-1.5.28 → dc_python_sdk-1.5.29}/src/dc_sdk/src/services/environment.py

@@ -27,6 +27,7 @@ class PipelineEnvironment:
     task_id: ClassVar[str] = None
     source_endpoint: ClassVar[str] = os.getenv("SOURCE_ENDPOINT") or "http://localhost:5000"
     destination_endpoint: ClassVar[str] = os.getenv("DESTINATION_ENDPOINT") or "http://localhost:5001"
+    pipeline_mapping_id: ClassVar[str] = os.getenv("PIPELINE_MAPPING_ID") or ""
 
     @staticmethod
     def validate_environment():