dc-python-sdk 1.5.42__tar.gz → 1.5.43__tar.gz

{dc_python_sdk-1.5.42/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.42
+Version: 1.5.43
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector
@@ -18,6 +18,7 @@ Requires-Dist: awslambdaric
 Requires-Dist: requests
 Requires-Dist: boto3>=1.40.0
 Requires-Dist: openai
+Requires-Dist: pycryptodome
 Provides-Extra: test
 Requires-Dist: python-dotenv>=0.20.0; extra == "test"
 Requires-Dist: faker>=13.12.0; extra == "test"

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.42"
+version = "1.5.43"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"
@@ -22,7 +22,8 @@ dependencies = [
     "awslambdaric",
     "requests",
     "boto3>=1.40.0",
-    "openai"
+    "openai",
+    "pycryptodome",
 ]
 
 [project.optional-dependencies]

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.42
+version = 1.5.43
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package
@@ -23,6 +23,7 @@ install_requires =
 	fastapi
 	uvicorn
 	awslambdaric
+	pycryptodome
 	requests
 	boto3>=1.40.0
 

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.42
+Version: 1.5.43
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector
@@ -18,6 +18,7 @@ Requires-Dist: awslambdaric
 Requires-Dist: requests
 Requires-Dist: boto3>=1.40.0
 Requires-Dist: openai
+Requires-Dist: pycryptodome
 Provides-Extra: test
 Requires-Dist: python-dotenv>=0.20.0; extra == "test"
 Requires-Dist: faker>=13.12.0; extra == "test"

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/src/dc_python_sdk.egg-info/requires.txt

@@ -4,6 +4,7 @@ awslambdaric
 requests
 boto3>=1.40.0
 openai
+pycryptodome
 
 [ai]
 openai

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/src/dc_sdk/src/models/pipeline_details.py

@@ -33,6 +33,8 @@ class PipelineDetails:
         self.options = json.loads(row_data['pipeline_object_options_json']) if 'pipeline_object_options_json' in row_data and row_data['pipeline_object_options_json'] else dict()
         self.max_allowed_retrieval = row_data.get('max_allowed_retrieval')
         self.primary_key_column_nm = row_data.get('primary_key_column_nm')
+        self.destination_credential_information = row_data.get('destination_credential_information')
+        self.source_credential_information = row_data.get('source_credential_information')
 
     def increment_stage(self):
         self.stage += 1

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/src/dc_sdk/src/pipeline.py

@@ -67,27 +67,54 @@ class PipelineConductor:
                 if self.authentication_tries == 3 or "firewall" in str(e): # todo: fix add retry flg in raised errors
                     raise e
                 self.authentication_tries += 1
-                self._refresh_credentials()
+                self._refresh_source_credentials()
                 time.sleep(10)
 
-        if authenticated and self.mode == "prod":
+        if authenticated:
             self.api.send_new_credentials(True, self.pipeline_id, self.connector.credentials)
             self.log(self.log_templates.AUTHENTICATION_FINISH.format(self.pipeline_details.source_credential_nm))
     
-    def _refresh_credentials(self):
+    def _refresh_source_credentials(self):
         response = self.api.get_credential_updates(True, self.pipeline_id)
 
-        creds = self.aws.decrypt_customer_data_object(response['credential'], response['organization'])
+        creds = self.aws.decrypt_customer_data_object(response['credential'], response['organization'], 
+            encrypted_data_key_txt=self.pipeline_details.source_credential_information.get('encrypted_data_key_txt'),
+            encryption_iv_txt=self.pipeline_details.source_credential_information.get('encryption_iv_txt'),
+            encryption_auth_tag_txt=self.pipeline_details.source_credential_information.get('encryption_auth_tag_txt')
+        )
         
         self.connector.credentials = creds
+
+    def _refresh_destination_credentials(self):
+        response = self.api.get_credential_updates(False, self.pipeline_id)
+
+        creds = self.aws.decrypt_customer_data_object(response['credential'], response['organization'], 
+            encrypted_data_key_txt=self.pipeline_details.destination_credential_information.get('encrypted_data_key_txt'),
+            encryption_iv_txt=self.pipeline_details.destination_credential_information.get('encryption_iv_txt'),
+            encryption_auth_tag_txt=self.pipeline_details.destination_credential_information.get('encryption_auth_tag_txt')
+        )
+        
+        self.connector.credentials = creds    
     
     def authenticate_destination(self):
         if not self.credentials:
             raise errors.AuthenticationError("Credentials are not set, please update your credentials.")
 
         self.log(self.log_templates.AUTHENTICATION_START.format(self.pipeline_details.destination_credential_nm, self.pipeline_details.destination_object_id))
-        self.connector.authenticate()
-        self.log(self.log_templates.AUTHENTICATION_FINISH.format(self.pipeline_details.destination_credential_nm))
+        authenticated = False
+        while self.authentication_tries <= 3 and not authenticated:
+            try:
+                authenticated = self.connector.authenticate()
+            except Exception as e:
+                if self.authentication_tries == 3 or "firewall" in str(e): # todo: fix add retry flg in raised errors
+                    raise e
+                self.authentication_tries += 1
+                self._refresh_destination_credentials()
+                time.sleep(10)
+
+        if authenticated:
+            self.api.send_new_credentials(False, self.pipeline_id, self.connector.credentials)
+            self.log(self.log_templates.AUTHENTICATION_FINISH.format(self.pipeline_details.destination_credential_nm))
 
     def get_data(self):
         # Determine batch size

dc_python_sdk-1.5.43/src/dc_sdk/src/services/api.py

@@ -0,0 +1,205 @@
+import os
+import requests
+import sys
+import json
+import time
+import random
+
+from ..models.enums import EnvironmentVariablesEnum
+from ..models.pipeline_details import PipelineDetails
+
+
+class DataConnectorAPI:
+    RETRY_STATUS_CODES = {408, 429, 500, 502, 503, 504}
+
+    def __init__(self, environment=None):
+        self.server_endpoint = os.environ.get(EnvironmentVariablesEnum.SERVER_ENDPOINT.value)
+        self.pipeline_id = os.environ.get(EnvironmentVariablesEnum.PIPELINE_ID.value)
+        self.server_api_key = os.environ.get(EnvironmentVariablesEnum.SERVER_API_KEY.value)
+        self.pipelines_base_url = f"{self.server_endpoint}/api/pipelines"
+
+    def get(self, endpoint):
+        return self._request("GET", endpoint)
+
+    def post(self, endpoint, body=None):
+        return self._request("POST", endpoint, body)
+
+    def put(self, endpoint, body=None):
+        return self._request("PUT", endpoint, body)
+
+    def log(self, message, stage, task, error=None, internal=False):
+        print("Log Output: ", message)
+
+        pipeline_run_history_id = os.environ.get(
+            EnvironmentVariablesEnum.PIPELINE_RUN_HISTORY_ID.value
+        )
+
+        payload = {
+            "PipelineID": self.pipeline_id,
+            "PipelineRunHistoryID": pipeline_run_history_id,
+            "LogStageID": stage,
+            "LogTXT": message,
+            "ExternalFacingFLG": 0 if internal else 1,
+            "ErrorLocationDSC": None,
+            "LogTypeCD": 2 if error else 1,
+            "LogTypeDSC": "Error" if error else "Info",
+            "task": task,
+        }
+
+        try:
+            self.post("log", payload)
+        except Exception as e:
+            print(f"WARNING: Failed to post pipeline log. Continuing. Error: {e}")
+
+    def logv2(self, logger_payload):
+        pipeline_run_history_id = os.environ.get(
+            EnvironmentVariablesEnum.PIPELINE_RUN_HISTORY_ID.value
+        )
+
+        payload = {
+            "PipelineID": self.pipeline_id,
+            "PipelineRunHistoryID": pipeline_run_history_id,
+            **logger_payload,
+        }
+
+        try:
+            self.post("logv2", payload)
+        except Exception as e:
+            print(f"WARNING: Failed to post pipeline logv2. Continuing. Error: {e}")
+
+    def get_pipeline_details(
+        self,
+        pipeline_id: str,
+        task,
+        pipeline_run_history_id: str,
+        pipeline_mapping_id: str = None,
+    ):
+        url = f"{pipeline_id}"
+
+        if pipeline_mapping_id:
+            url = f"{url}?PipelineMappingID={pipeline_mapping_id}"
+
+        response_json = self.get(url)
+
+        return PipelineDetails(response_json, task, pipeline_id, pipeline_run_history_id)
+
+    def create_new_history(self, pipeline_id, pipeline_mapping_id: str = None):
+        body = {"PipelineMappingID": pipeline_mapping_id} if pipeline_mapping_id else None
+        response = self.post(f"{pipeline_id}/history", body)
+
+        return response["PipelineRunHistoryID"]
+
+    def send_new_credentials(self, source_flg, pipeline_id, credentials):
+        self.post(
+            f"credentials/{pipeline_id}",
+            {
+                "source": source_flg,
+                "credentials": json.dumps(credentials),
+            },
+        )
+
+    def get_credential_updates(self, source_flg, pipeline_id):
+        response = self.get(
+            f"credentials/{pipeline_id}?source={'true' if source_flg else 'false'}"
+        )
+
+        return response
+
+    def create_pipeline_mapping(self, pipeline_id, pipeline_mapping_json):
+        body = {
+            "pipeline_id": pipeline_id,
+            "pipeline_mapping_json": pipeline_mapping_json,
+        }
+
+        self.post(f"{pipeline_id}/mapping", body)
+
+    def _request(self, method, endpoint, body=None, max_attempts=4):
+        endpoint = endpoint.lstrip("/")
+        url = f"{self.pipelines_base_url}/{endpoint}"
+
+        headers = {
+            "Accept": "application/json",
+            "x-data-connector-access-token": self.server_api_key,
+        }
+
+        last_error = None
+
+        for attempt in range(1, max_attempts + 1):
+            try:
+                response = requests.request(
+                    method=method,
+                    url=url,
+                    json=body,
+                    headers=headers,
+                    timeout=(5, 30),
+                )
+
+                try:
+                    response_data = response.json()
+                except ValueError:
+                    response_data = response.text
+
+                if response.ok:
+                    return response_data
+
+                if response.status_code in self.RETRY_STATUS_CODES and attempt < max_attempts:
+                    sleep_seconds = self._get_retry_delay(attempt)
+                    print(
+                        f"WARNING: API {method} {endpoint} failed with "
+                        f"{response.status_code}. Retrying attempt "
+                        f"{attempt + 1}/{max_attempts} in {sleep_seconds:.2f}s."
+                    )
+                    time.sleep(sleep_seconds)
+                    continue
+
+                raise Exception(
+                    f"API ERROR: {method} {endpoint} failed | "
+                    f"Status: {response.status_code} | "
+                    f"Response: {response_data}"
+                )
+
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+                last_error = e
+
+                if attempt < max_attempts:
+                    sleep_seconds = self._get_retry_delay(attempt)
+                    print(
+                        f"WARNING: API {method} {endpoint} connection/timeout error. "
+                        f"Retrying attempt {attempt + 1}/{max_attempts} "
+                        f"in {sleep_seconds:.2f}s. Error: {e}"
+                    )
+                    time.sleep(sleep_seconds)
+                    continue
+
+                raise Exception(
+                    f"API ERROR: {method} {endpoint} failed after "
+                    f"{max_attempts} attempts | Error: {str(e)}"
+                )
+
+            except requests.exceptions.RequestException as e:
+                last_error = e
+
+                if attempt < max_attempts:
+                    sleep_seconds = self._get_retry_delay(attempt)
+                    print(
+                        f"WARNING: API {method} {endpoint} unexpected request error. "
+                        f"Retrying attempt {attempt + 1}/{max_attempts} "
+                        f"in {sleep_seconds:.2f}s. Error: {e}"
+                    )
+                    time.sleep(sleep_seconds)
+                    continue
+
+                raise Exception(
+                    f"API ERROR: {method} {endpoint} unexpected error after "
+                    f"{max_attempts} attempts | Error: {str(e)}"
+                )
+
+        raise Exception(
+            f"API ERROR: {method} {endpoint} failed after "
+            f"{max_attempts} attempts | Last error: {str(last_error)}"
+        )
+
+    def _get_retry_delay(self, attempt):
+        base_delay = min(2 ** attempt, 10)
+        jitter = random.uniform(0, 0.5)
+        return base_delay + jitter

dc_python_sdk-1.5.43/src/dc_sdk/src/services/aws.py

@@ -0,0 +1,283 @@
+import os
+import boto3
+import json
+from typing import Dict
+from base64 import b64decode
+from Crypto.Cipher import AES
+
+kms_client = boto3.client('kms')
+
+APP_ENV = os.environ.get("APP_ENV")
+
+s3_client = boto3.client('s3')
+s3_resource = boto3.resource('s3')
+ecs_resource = boto3.client('ecs')
+kms_client = boto3.client('kms')
+
+class AwsService:
+    def __init__(self, s3_bucket) -> None:
+        self.s3_bucket = s3_bucket
+        self._ecs_cluster_name = APP_ENV if APP_ENV != "local" else "development"
+
+    def get_keys(self, pipeline_run_history_id):
+        keys = []
+        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"transfers/e{pipeline_run_history_id}")
+        if 'Contents' in response:
+            for key in response['Contents']:
+                keys.append(key['Key'])
+        return keys
+
+    def get_dev_keys(self, prefix):
+        keys = []
+        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"devTransfers/e{prefix}")
+        if 'Contents' in response:
+            for key in response['Contents']:
+                keys.append(key['Key'])
+        return keys
+
+    def download_object(self, key_name):
+        file = s3_resource.Object(self.s3_bucket, key_name)
+        return file.get()['Body']
+
+    def upload_object(self, key_name, json_buffer=None, file_path=None):
+        if json_buffer is not None:
+            s3_resource.Object(
+                self.s3_bucket, key_name).put(Body=json_buffer.getvalue())
+        elif file_path is not None:
+            s3_client.upload_file(file_path, self.s3_bucket, key_name)
+        else:
+            raise ValueError("Either json_buffer or file_path must be provided")
+
+    def get_object_size(self, key_name):
+        response = s3_client.head_object(
+            Bucket=self.s3_bucket, Key=key_name)
+        return float(response['ContentLength'])
+
+    def delete_object(self, key_name):
+        s3_client.delete_object(
+            Bucket=self.s3_bucket,
+            Key=key_name,
+        )
+
+    def start_ecs_task(self, task_name:str, task_version:str, pipeline_id: str, pipeline_run_history_id: str):
+        response = ecs_resource.run_task(
+            cluster=self._ecs_cluster_name,
+            count=1,
+            enableECSManagedTags=False,
+            enableExecuteCommand=False,
+            launchType='FARGATE',
+            networkConfiguration={
+                'awsvpcConfiguration': {
+                    'subnets': [
+                        'subnet-095c3230c890e0841',
+                    ],
+                    'securityGroups': [
+                        'sg-08d46d487577cbdf9',
+                    ],
+                    'assignPublicIp': 'ENABLED'
+                }
+            },
+            overrides={
+                'containerOverrides': [
+                    {
+                        'name': task_name,
+                        'command': ['app.py'],
+                        'environment': [
+                            {
+                                'name': 'PIPELINE_ID',
+                                'value': pipeline_id
+                            },
+                            {
+                                'name': 'PIPELINE_RUN_HISTORY_ID',
+                                'value': pipeline_run_history_id
+                            },
+                            {
+                                'name': 'TASK',
+                                'value': 'DESTINATION'
+                            },
+                            {
+                                'name': 'APP_ENV',
+                                'value': os.environ.get("APP_ENV")
+                            },
+                            {
+                                'name': 'SERVER_ENDPOINT',
+                                'value': os.environ.get("SERVER_ENDPOINT")
+                            },
+                            {
+                                'name': 'SERVER_API_KEY',
+                                'value': os.environ.get("SERVER_API_KEY")
+                            },
+                            {
+                                'name': 'AWS_DEFAULT_REGION',
+                                'value': os.environ.get("AWS_DEFAULT_REGION")
+                            }
+                        ],
+                    },
+                ],
+            },
+            platformVersion='LATEST',
+            taskDefinition=f"{task_name}:{task_version}"
+        )
+
+        if len(response['tasks']) == 0:
+            raise Exception("Failed to start next connector")
+
+        return response['tasks'][0]['taskArn'].split("/")[-1]
+
+    def decrypt_customer_data(
+        self,
+        encrypted_data: str,
+        customer_id: str,
+        encrypted_data_key_txt: str = None,
+        encryption_iv_txt: str = None,
+        encryption_auth_tag_txt: str = None
+    ) -> str:
+        """Decrypt data with customer context using legacy or envelope encryption"""
+        encryption_service = EncryptionService()
+
+        return encryption_service.decrypt_customer_data(
+            encrypted_data=encrypted_data,
+            customer_id=customer_id,
+            encrypted_data_key_txt=encrypted_data_key_txt,
+            encryption_iv_txt=encryption_iv_txt,
+            encryption_auth_tag_txt=encryption_auth_tag_txt
+        )
+
+
+    def decrypt_customer_data_object(
+        self,
+        encrypted_data: str,
+        customer_id: str,
+        encrypted_data_key_txt: str = None,
+        encryption_iv_txt: str = None,
+        encryption_auth_tag_txt: str = None
+    ) -> Dict:
+        """Decrypt JSON data with customer context using legacy or envelope encryption"""
+        encryption_service = EncryptionService()
+
+        return encryption_service.decrypt_customer_data_object(
+            encrypted_data=encrypted_data,
+            customer_id=customer_id,
+            encrypted_data_key_txt=encrypted_data_key_txt,
+            encryption_iv_txt=encryption_iv_txt,
+            encryption_auth_tag_txt=encryption_auth_tag_txt
+        )
+
+
+
+
+
+class EncryptionService:
+    def decrypt_customer_data(
+        self,
+        encrypted_data: str,
+        customer_id: str,
+        encrypted_data_key_txt: str = None,
+        encryption_iv_txt: str = None,
+        encryption_auth_tag_txt: str = None
+    ) -> str:
+        """
+        Decrypt customer data using either:
+        - legacy KMS direct encryption
+        - envelope encryption
+        """
+
+        try:
+            is_envelope = (
+                encrypted_data_key_txt is not None and
+                encryption_iv_txt is not None and
+                encryption_auth_tag_txt is not None
+            )
+
+            if is_envelope:
+                return self._decrypt_customer_data_envelope(
+                    encrypted_data=encrypted_data,
+                    customer_id=customer_id,
+                    encrypted_data_key_txt=encrypted_data_key_txt,
+                    encryption_iv_txt=encryption_iv_txt,
+                    encryption_auth_tag_txt=encryption_auth_tag_txt
+                )
+
+            return self._decrypt_customer_data_legacy(
+                encrypted_data=encrypted_data,
+                customer_id=customer_id
+            )
+
+        except Exception as e:
+            raise Exception(f'Decryption failed: {str(e)}')
+
+    def _decrypt_customer_data_legacy(
+        self,
+        encrypted_data: str,
+        customer_id: str
+    ) -> str:
+
+        response = kms_client.decrypt(
+            CiphertextBlob=b64decode(encrypted_data.encode('utf-8')),
+            EncryptionContext={
+                'CustomerId': customer_id
+            }
+        )
+
+        if 'Plaintext' not in response:
+            raise Exception('Legacy decryption failed')
+
+        return response['Plaintext'].decode('utf-8')
+
+    def _decrypt_customer_data_envelope(
+        self,
+        encrypted_data: str,
+        customer_id: str,
+        encrypted_data_key_txt: str,
+        encryption_iv_txt: str,
+        encryption_auth_tag_txt: str
+    ) -> str:
+
+        # decrypt data key using KMS
+        data_key_response = kms_client.decrypt(
+            CiphertextBlob=b64decode(
+                encrypted_data_key_txt.encode('utf-8')
+            ),
+            EncryptionContext={
+                'CustomerId': customer_id
+            }
+        )
+
+        if 'Plaintext' not in data_key_response:
+            raise Exception('Data key decryption failed')
+
+        plaintext_data_key = data_key_response['Plaintext']
+
+        cipher = AES.new(
+            plaintext_data_key,
+            AES.MODE_GCM,
+            nonce=b64decode(encryption_iv_txt.encode('utf-8'))
+        )
+
+        cipher.update(b'')
+
+        decrypted_data = cipher.decrypt_and_verify(
+            b64decode(encrypted_data.encode('utf-8')),
+            b64decode(encryption_auth_tag_txt.encode('utf-8'))
+        )
+
+        return decrypted_data.decode('utf-8')
+
+    def decrypt_customer_data_object(
+        self,
+        encrypted_data: str,
+        customer_id: str,
+        encrypted_data_key_txt: str = None,
+        encryption_iv_txt: str = None,
+        encryption_auth_tag_txt: str = None
+    ) -> dict:
+
+        data = self.decrypt_customer_data(
+            encrypted_data=encrypted_data,
+            customer_id=customer_id,
+            encrypted_data_key_txt=encrypted_data_key_txt,
+            encryption_iv_txt=encryption_iv_txt,
+            encryption_auth_tag_txt=encryption_auth_tag_txt
+        )
+
+        return json.loads(data)

{dc_python_sdk-1.5.42 → dc_python_sdk-1.5.43}/src/dc_sdk/src/services/environment.py

@@ -78,6 +78,10 @@ class PipelineEnvironment:
 
     @staticmethod
     def get_task_id():
+        explicit = os.getenv("DC_TASK_ID")
+        if explicit:
+            return explicit
+
         metadata_uri = os.getenv("ECS_CONTAINER_METADATA_URI_V4")
 
         # Not running in ECS

dc_python_sdk-1.5.42/src/dc_sdk/src/services/api.py

@@ -1,123 +0,0 @@
-import os, requests, sys, json
-from ..models.enums import EnvironmentVariablesEnum
-from ..models.pipeline_details import PipelineDetails
-
-class DataConnectorAPI:
-    def __init__(self, environment = None):
-        self.server_endpoint = os.environ.get(EnvironmentVariablesEnum.SERVER_ENDPOINT.value)
-        self.pipeline_id = os.environ.get(EnvironmentVariablesEnum.PIPELINE_ID.value)
-        self.server_api_key = os.environ.get(EnvironmentVariablesEnum.SERVER_API_KEY.value)
-        self.pipelines_base_url = f"{self.server_endpoint}/api/pipelines"
-
-    def get(self, endpoint):
-        return self._request("GET", endpoint)
-
-    def post(self, endpoint, body=None):
-        return self._request("POST", endpoint, body)
-
-    def put(self, endpoint, body=None):
-        return self._request("PUT", endpoint, body)
-
-    def log(self, message, stage, task, error=None, internal=False):
-        print("Log Output: ", message)
-        PIPELINE_RUN_HISTORY_ID = os.environ.get(EnvironmentVariablesEnum.PIPELINE_RUN_HISTORY_ID.value)
-        payload = {
-            'PipelineID': self.pipeline_id,
-            'PipelineRunHistoryID': PIPELINE_RUN_HISTORY_ID,
-            'LogStageID': stage,
-            'LogTXT': message,
-            'ExternalFacingFLG': 0 if internal else 1,
-            'ErrorLocationDSC': None, # TODO: UPDATE THIS
-            'LogTypeCD': 2 if error else 1,
-            'LogTypeDSC': 'Error' if error else 'Info',
-            'task': task,
-        }
-
-        self.post('log', payload)
-
-    def logv2(self, logger_payload):
-        PIPELINE_RUN_HISTORY_ID = os.environ.get(EnvironmentVariablesEnum.PIPELINE_RUN_HISTORY_ID.value)
-        payload = {
-            'PipelineID': self.pipeline_id,
-            'PipelineRunHistoryID': PIPELINE_RUN_HISTORY_ID,
-            **logger_payload
-        }
-
-        self.post('logv2', payload)
-
-    def get_pipeline_details(self, pipeline_id: str, task, pipeline_run_history_id: str, pipeline_mapping_id: str = None):
-        url = f"{pipeline_id}"
-        if pipeline_mapping_id:
-            url = f"{url}?PipelineMappingID={pipeline_mapping_id}"
-        json = self.get(url)
-
-        return PipelineDetails(json, task, pipeline_id, pipeline_run_history_id)
-
-    def create_new_history(self, pipeline_id, pipeline_mapping_id: str = None):
-        if pipeline_mapping_id:
-            body = {"PipelineMappingID": pipeline_mapping_id}
-        else:
-            body = None
-        response = self.post(f"{pipeline_id}/history", body)
-
-        return response['PipelineRunHistoryID']
-
-    def send_new_credentials(self, source_flg, pipeline_id, credentials):
-        self.post(f"credentials/{pipeline_id}", {"source": source_flg, "credentials": json.dumps(credentials)})
-
-    def get_credential_updates(self, source_flg, pipeline_id):
-        response = self.get(f"credentials/{pipeline_id}?source={'true' if source_flg else 'false'}")
-
-        return response
-
-    def create_pipeline_mapping(self, pipeline_id, pipeline_mapping_json):
-        body = {
-            "pipeline_id": pipeline_id,
-            "pipeline_mapping_json": pipeline_mapping_json
-        }
-        self.post(f"/{pipeline_id}/mapping", body)
-
-    def _request(self, method, endpoint, body=None):
-        url = f"{self.pipelines_base_url}/{endpoint}"
-
-        try:
-            response = requests.request(
-                method=method,
-                url=url,
-                data=body,
-                headers={
-                    'Accept': 'application/json',
-                    'x-data-connector-access-token': self.server_api_key
-                },
-                timeout=10  # 🔥 IMPORTANT
-            )
-
-            # Try to parse JSON safely
-            try:
-                response_data = response.json()
-            except ValueError:
-                response_data = response.text
-
-            if response.ok:
-                return response_data
-            else:
-                raise Exception(
-                    f"API ERROR: {method} {endpoint} failed | "
-                    f"Status: {response.status_code} | "
-                    f"Response: {response_data}"
-                )
-
-        except requests.exceptions.Timeout:
-            raise Exception(
-                f"API ERROR: {method} {endpoint} timed out (10s)"
-            )
-
-        except requests.exceptions.ConnectionError:
-            raise Exception(
-                f"API ERROR: {method} {endpoint} failed — no response from server"
-            )
-
-        except requests.exceptions.RequestException as e:
-            raise Exception(
-                f"API ERROR: {method} {endpoint} unexpected error | {str(e)}"
-            )

dc_python_sdk-1.5.42/src/dc_sdk/src/services/aws.py

@@ -1,141 +0,0 @@
-import os
-import boto3
-import json
-from typing import Dict
-from base64 import b64decode
-
-APP_ENV = os.environ.get("APP_ENV")
-
-s3_client = boto3.client('s3')
-s3_resource = boto3.resource('s3')
-ecs_resource = boto3.client('ecs')
-kms_client = boto3.client('kms')
-
-class AwsService:
-    def __init__(self, s3_bucket) -> None:
-        self.s3_bucket = s3_bucket
-        self._ecs_cluster_name = APP_ENV if APP_ENV != "local" else "development"
-
-    def get_keys(self, pipeline_run_history_id):
-        keys = []
-        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"transfers/e{pipeline_run_history_id}")
-        if 'Contents' in response:
-            for key in response['Contents']:
-                keys.append(key['Key'])
-        return keys
-
-    def get_dev_keys(self, prefix):
-        keys = []
-        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"devTransfers/e{prefix}")
-        if 'Contents' in response:
-            for key in response['Contents']:
-                keys.append(key['Key'])
-        return keys
-
-    def download_object(self, key_name):
-        file = s3_resource.Object(self.s3_bucket, key_name)
-        return file.get()['Body']
-
-    def upload_object(self, key_name, json_buffer=None, file_path=None):
-        if json_buffer is not None:
-            s3_resource.Object(
-                self.s3_bucket, key_name).put(Body=json_buffer.getvalue())
-        elif file_path is not None:
-            s3_client.upload_file(file_path, self.s3_bucket, key_name)
-        else:
-            raise ValueError("Either json_buffer or file_path must be provided")
-
-    def get_object_size(self, key_name):
-        response = s3_client.head_object(
-            Bucket=self.s3_bucket, Key=key_name)
-        return float(response['ContentLength'])
-
-    def delete_object(self, key_name):
-        s3_client.delete_object(
-            Bucket=self.s3_bucket,
-            Key=key_name,
-        )
-
-    def start_ecs_task(self, task_name:str, task_version:str, pipeline_id: str, pipeline_run_history_id: str):
-        response = ecs_resource.run_task(
-            cluster=self._ecs_cluster_name,
-            count=1,
-            enableECSManagedTags=False,
-            enableExecuteCommand=False,
-            launchType='FARGATE',
-            networkConfiguration={
-                'awsvpcConfiguration': {
-                    'subnets': [
-                        'subnet-095c3230c890e0841',
-                    ],
-                    'securityGroups': [
-                        'sg-08d46d487577cbdf9',
-                    ],
-                    'assignPublicIp': 'ENABLED'
-                }
-            },
-            overrides={
-                'containerOverrides': [
-                    {
-                        'name': task_name,
-                        'command': ['app.py'],
-                        'environment': [
-                            {
-                                'name': 'PIPELINE_ID',
-                                'value': pipeline_id
-                            },
-                            {
-                                'name': 'PIPELINE_RUN_HISTORY_ID',
-                                'value': pipeline_run_history_id
-                            },
-                            {
-                                'name': 'TASK',
-                                'value': 'DESTINATION'
-                            },
-                            {
-                                'name': 'APP_ENV',
-                                'value': os.environ.get("APP_ENV")
-                            },
-                            {
-                                'name': 'SERVER_ENDPOINT',
-                                'value': os.environ.get("SERVER_ENDPOINT")
-                            },
-                            {
-                                'name': 'SERVER_API_KEY',
-                                'value': os.environ.get("SERVER_API_KEY")
-                            },
-                            {
-                                'name': 'AWS_DEFAULT_REGION',
-                                'value': os.environ.get("AWS_DEFAULT_REGION")
-                            }
-                        ],
-                    },
-                ],
-            },
-            platformVersion='LATEST',
-            taskDefinition=f"{task_name}:{task_version}"
-        )
-
-        if len(response['tasks']) == 0:
-            raise Exception("Failed to start next connector")
-
-        return response['tasks'][0]['taskArn'].split("/")[-1]
-
-    def decrypt_customer_data(self, encrypted_data: str, customer_id: str) -> str:
-        """Decrypt data with customer context"""
-        try:
-            response = kms_client.decrypt(
-                CiphertextBlob=b64decode(encrypted_data.encode('utf-8')),
-                EncryptionContext={'CustomerId': customer_id}
-            )
-            if 'Plaintext' not in response:
-                raise Exception('Decryption failed')
-            
-            return response['Plaintext'].decode('utf-8')
-        except Exception as e:
-            raise Exception(f'Decryption failed: {str(e)}')
-
-    def decrypt_customer_data_object(self, encrypted_data: str, customer_id: str) -> Dict:
-        """Decrypt JSON data with customer context"""
-        data = self.decrypt_customer_data(encrypted_data, customer_id)
-        return json.loads(data)