dc-python-sdk 1.5.48__tar.gz → 1.5.50__tar.gz

{dc_python_sdk-1.5.48/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.48
+Version: 1.5.50
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.48"
+version = "1.5.50"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.48
+version = 1.5.50
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.48
+Version: 1.5.50
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_python_sdk.egg-info/SOURCES.txt

@@ -11,12 +11,15 @@ src/dc_python_sdk.egg-info/top_level.txt
 src/dc_sdk/__init__.py
 src/dc_sdk/app.py
 src/dc_sdk/cli.py
+src/dc_sdk/data_stream.py
 src/dc_sdk/errors.py
 src/dc_sdk/handler.py
 src/dc_sdk/types.py
 src/dc_sdk/src/__init__.py
 src/dc_sdk/src/ai.py
 src/dc_sdk/src/ai_http.py
+src/dc_sdk/src/connection_status.py
+src/dc_sdk/src/destination_object_template.py
 src/dc_sdk/src/mapping.py
 src/dc_sdk/src/pipeline.py
 src/dc_sdk/src/server.py

dc_python_sdk-1.5.50/src/dc_sdk/__init__.py

@@ -0,0 +1,3 @@
+from dc_sdk.data_stream import DataStream
+
+__all__ = ["DataStream"]

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_sdk/cli.py

@@ -18,8 +18,6 @@ def main():
 
     print("version: ", version("dc-python-sdk"))
 
-    print(f"[DC SDK] Command: {command}")
-
     if command == "http":
         from dc_sdk.src.server import start_server
         start_server()

dc_python_sdk-1.5.50/src/dc_sdk/data_stream.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, Optional
+
+
+@dataclass
+class DataStream:
+    """Byte stream returned by connector get_data_stream for transfer/upload."""
+
+    stream: Iterable[bytes]
+
+    # What are the bytes?
+    media_type: str
+
+    # Optional processing hints
+    container: Optional[str] = None  # zip, gzip, tar
+    format: Optional[str] = None  # csv, ndjson, parquet, xml, xlsx
+
+    # Optional metadata
+    filename: Optional[str] = None
+    encoding: Optional[str] = None
+
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def hints(self) -> Dict[str, Any]:
+        """Serializable extract/normalize hints (excludes the live stream)."""
+        return {
+            "media_type": self.media_type,
+            "container": self.container,
+            "format": self.format,
+            "filename": self.filename,
+            "encoding": self.encoding,
+            "metadata": self.metadata,
+        }

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_sdk/handler.py

@@ -1,5 +1,6 @@
 from dc_sdk.errors import Error
 from dc_sdk.src.mapping import Mapping
+from dc_sdk.src.connection_status import STATUS_ERROR
 import traceback
 from importlib.metadata import version
 
@@ -9,7 +10,9 @@ def get_action_name(action: int) -> str:
         1: "retrieving fields",
         2: "retrieving 5 row preview",
         3: "testing connection",
-        4: "starting the connector"
+        4: "starting the connector",
+        5: "retrieving metadata",
+        6: "retrieving objects",
     }[action]
 
 def apply_data_filters(results, data_filters):
@@ -159,6 +162,9 @@ def handler(event, context):
     mapping = None
     action = int(event['action']) if 'action' in event else 4
     get_objects = event.get('get_objects', True)
+    skip_authenticate = event.get('skip_authenticate', False)
+    force_authenticate = event.get('force_authenticate', False)
+    include_metadata = event.get('include_metadata', True)
     credentials_dict = event['credentials'] if 'credentials' in event else None
     object_id = event['object_id'] if 'object_id' in event else None
     field_ids = event['mapping'] if 'mapping' in event else None
@@ -174,9 +180,14 @@ def handler(event, context):
 
         if action == 0:
             if get_objects:
-                results, message = mapping.connect_get_objects()
+                results, message = mapping.connect_get_objects(
+                    skip_authenticate=skip_authenticate,
+                    force_authenticate=force_authenticate,
+                )
             else:
-                results, message = mapping.authenticate()
+                results, message = mapping.authenticate(
+                    force_authenticate=force_authenticate or not skip_authenticate,
+                )
         elif action == 1:
             results, message = mapping.get_fields(object_id, options)
         elif action == 2:
@@ -185,6 +196,17 @@ def handler(event, context):
             results = apply_data_filters(results, data_filters)
         elif action == 3:
             results, message = mapping.test_connection()
+        elif action == 5:
+            results, message = mapping.get_metadata_only(
+                skip_authenticate=skip_authenticate,
+                force_authenticate=force_authenticate,
+            )
+        elif action == 6:
+            results, message = mapping.get_objects_only(
+                skip_authenticate=skip_authenticate,
+                force_authenticate=force_authenticate,
+                include_metadata=include_metadata,
+            )
         else:
             raise Error("Invalid action", "InvalidActionError")
     except Error as e:
@@ -192,19 +214,28 @@ def handler(event, context):
         internal_error = e.internal
         error_trace = traceback.format_exc()
         error = error_trace
+        if mapping:
+            mapping._set_connection_status(STATUS_ERROR)
 
     except Exception as e:
         error_trace = traceback.format_exc()
         error = error_trace
         internal_error = True
+        if mapping:
+            mapping._set_connection_status(STATUS_ERROR)
 
     if error:
         print(error)
 
+    last_status_dsc = None
+    if mapping and mapping.connector.credentials:
+        last_status_dsc = mapping.connector.credentials.get("last_status_dsc")
+
     return {
         'message': message if not internal_error else f"Something went wrong with {action_name}",
         'results': results,
         'credentials': mapping.connector.credentials if mapping else None,
+        'last_status_dsc': last_status_dsc,
         'error': error,
         'internal_error': internal_error
     }

dc_python_sdk-1.5.50/src/dc_sdk/src/connection_status.py

@@ -0,0 +1,34 @@
+STATUS_CONNECTED = "Connected"
+STATUS_ERROR = "Error"
+STATUS_UNKNOWN = "Unknown"
+STATUS_NOT_SETUP = "NotSetup"
+STATUS_PENDING = "Pending"
+
+
+def normalize_status(status):
+    if not status:
+        return STATUS_UNKNOWN
+
+    normalized = str(status).strip()
+    lowered = normalized.lower()
+
+    if lowered in ("connected", "success"):
+        return STATUS_CONNECTED
+    if lowered in ("error", "failed", "failed to connect"):
+        return STATUS_ERROR
+    if lowered in ("notsetup", "not_setup", "not setup"):
+        return STATUS_NOT_SETUP
+    if lowered == "pending":
+        return STATUS_PENDING
+    if lowered == "unknown":
+        return STATUS_UNKNOWN
+
+    return normalized
+
+
+def should_skip_authenticate(status):
+    return normalize_status(status) == STATUS_CONNECTED
+
+
+def status_from_authenticated(authenticated):
+    return STATUS_CONNECTED if authenticated else STATUS_ERROR

dc_python_sdk-1.5.50/src/dc_sdk/src/destination_object_template.py

@@ -0,0 +1,74 @@
+"""Resolve dynamic tokens in destination object paths / file names at run time."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Optional
+
+_TOKEN_RE = re.compile(r"\{([A-Za-z0-9_-]+)\}")
+
+_DATE_FORMATS = {
+    "YYYYMMDD": "%Y%m%d",
+    "YYYY-MM-DD": "%Y-%m-%d",
+    "YYMMDD": "%y%m%d",
+    "HHMMSS": "%H%M%S",
+    "YYYYMMDDHHMMSS": "%Y%m%d%H%M%S",
+}
+
+
+def _slug(value: Optional[str]) -> str:
+    if value is None:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    text = re.sub(r"[^A-Za-z0-9]+", "_", text)
+    return text.strip("_")
+
+
+def resolve_destination_object_template(
+    template: Optional[str],
+    *,
+    source_object_id: Optional[str] = None,
+    source_connector_nm: Optional[str] = None,
+    when: Optional[datetime] = None,
+) -> str:
+    """
+    Expand `{YYYYMMDD}`, `{object}`, `{connector}`, etc. in a destination path.
+
+    Unknown tokens are left unchanged so mis-typed braces remain visible in logs.
+    """
+    if template is None:
+        return ""
+    value = str(template)
+    if "{" not in value:
+        return value
+
+    now = when or datetime.now(timezone.utc)
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=timezone.utc)
+
+    replacements = {
+        "object": _slug(source_object_id),
+        "connector": _slug(source_connector_nm),
+    }
+    for token, fmt in _DATE_FORMATS.items():
+        replacements[token] = now.strftime(fmt)
+
+    def _replace(match: re.Match) -> str:
+        key = match.group(1)
+        if key in replacements:
+            return replacements[key]
+        # Autocomplete may insert uppercase tokens; users often type lowercase
+        # (`{hhmmss}`). Resolve date formats case-insensitively, and
+        # `{object}` / `{connector}` case-insensitively.
+        upper = key.upper()
+        if upper in _DATE_FORMATS:
+            return replacements[upper]
+        lower = key.lower()
+        if lower in ("object", "connector"):
+            return replacements[lower]
+        return match.group(0)
+
+    return _TOKEN_RE.sub(_replace, value)

dc_python_sdk-1.5.50/src/dc_sdk/src/mapping.py

@@ -0,0 +1,191 @@
+from dc_sdk import errors
+import logging
+from dc_sdk.src.services.loader import load_connector
+from dc_sdk.src.connection_status import (
+    STATUS_CONNECTED,
+    STATUS_ERROR,
+    should_skip_authenticate,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class Mapping():
+    def __init__(self, credentials):
+        Connector = load_connector()
+        self.connector = Connector(credentials)
+
+    def _set_connection_status(self, status):
+        if self.connector.credentials is None:
+            self.connector.credentials = {}
+        self.connector.credentials["last_status_dsc"] = status
+
+    def _initialize_connector_session(self):
+        init = getattr(self.connector, "ensure_initialized", None)
+        if callable(init):
+            init()
+            return
+
+        legacy_init = getattr(self.connector, "_ensure_initialized", None)
+        if callable(legacy_init):
+            legacy_init()
+            return
+
+        if not self.connector.authenticate():
+            raise errors.AuthenticationError(message="Failed to initialize connection session")
+
+    def _ensure_session(self, force_authenticate=False):
+        credentials = self.connector.credentials or {}
+        status = credentials.get("last_status_dsc")
+
+        if not force_authenticate and should_skip_authenticate(status):
+            try:
+                self._initialize_connector_session()
+                return True
+            except Exception:
+                self._set_connection_status(STATUS_ERROR)
+                raise
+
+        if self.connector.authenticate():
+            self._set_connection_status(STATUS_CONNECTED)
+            return True
+
+        self._set_connection_status(STATUS_ERROR)
+        raise errors.AuthenticationError(message="Failed to authenticate")
+
+    def _resolve_session(self, skip_authenticate=False, force_authenticate=False):
+        if force_authenticate:
+            self._ensure_session(force_authenticate=True)
+            return
+
+        credentials = self.connector.credentials or {}
+        status = credentials.get("last_status_dsc")
+
+        if skip_authenticate and should_skip_authenticate(status):
+            self._ensure_session(force_authenticate=False)
+            return
+
+        if skip_authenticate and not should_skip_authenticate(status):
+            self._ensure_session(force_authenticate=True)
+            return
+
+        self._ensure_session(force_authenticate=False)
+
+    def _safe_get_metadata(self):
+        try:
+            return self.connector.get_metadata()
+        except errors.NotImplementedError:
+            return None
+        except Exception:
+            logger.exception("Error getting metadata")
+            return None
+
+    def authenticate(self, force_authenticate=True):
+        results = None
+        message = None
+
+        self._ensure_session(force_authenticate=force_authenticate)
+
+        metadata = self._safe_get_metadata()
+        results = {
+            "metadata": metadata
+        }
+        message = "Authenticated successfully"
+
+        return [results, message]
+
+    def connect_get_objects(self, skip_authenticate=False, force_authenticate=False):
+        results = None
+        metadata = None
+        objects = None
+        message = None
+
+        self._resolve_session(
+            skip_authenticate=skip_authenticate,
+            force_authenticate=force_authenticate,
+        )
+
+        objects = self.connector.get_objects()
+        metadata = self._safe_get_metadata()
+
+        results = {
+            "metadata": metadata,
+            "objects": objects,
+        }
+        message = "Retrieved objects"
+
+        return [results, message]
+
+    def get_metadata_only(self, skip_authenticate=False, force_authenticate=False):
+        self._resolve_session(
+            skip_authenticate=skip_authenticate,
+            force_authenticate=force_authenticate,
+        )
+
+        metadata = self._safe_get_metadata()
+        return [{"metadata": metadata}, "Retrieved metadata"]
+
+    def get_objects_only(self, skip_authenticate=False, force_authenticate=False, include_metadata=True):
+        self._resolve_session(
+            skip_authenticate=skip_authenticate,
+            force_authenticate=force_authenticate,
+        )
+
+        objects = self.connector.get_objects()
+        results = {"objects": objects}
+
+        if include_metadata:
+            results["metadata"] = self._safe_get_metadata()
+
+        return [results, "Retrieved objects"]
+
+    def test_connection(self):
+        results = None
+        message = None
+
+        self._ensure_session(force_authenticate=True)
+        message = "Authenticated successfully"
+
+        return [results, message]
+
+    def get_fields(self, object_id, options):
+        results = None
+        message = None
+        self._ensure_session(force_authenticate=False)
+        results = self.connector.get_fields(object_id, options=options)
+        message = "Retrieved fields"
+
+        return [results, message]
+
+    def get_five_row_preview(self, object_id, field_ids, options, n_rows=5, filters=None, next_page=None):
+        results = None
+        message = None
+
+        self._ensure_session(force_authenticate=False)
+        results = self.connector.get_data(
+            object_id,
+            field_ids=field_ids,
+            n_rows=n_rows,
+            filters=filters,
+            next_page=next_page,
+            options=options,
+        )
+        message = "Retrieved 5 row preview"
+
+        return [results, message]
+
+    def get_data(self, object_id, field_ids, n_rows, next_page, options):
+        results = None
+        message = None
+
+        self._ensure_session(force_authenticate=False)
+        results = self.connector.get_data(
+            object_id,
+            field_ids=field_ids,
+            n_rows=n_rows,
+            next_page=next_page,
+            options=options,
+        )
+        message = "Retrieved data"
+
+        return [results, message]

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_sdk/src/pipeline.py

@@ -1,4 +1,5 @@
 import json, io, math, re, inspect
+import os
 import time
 from .services.environment import PipelineEnvironment
 from .services.api import DataConnectorAPI
@@ -6,10 +7,26 @@ from .models.log_templates import LogTemplates
 from .services.aws import AwsService
 from .models import errors
 from .services.loader import load_connector
+from .destination_object_template import resolve_destination_object_template
+from dc_sdk.data_stream import DataStream
 
 TEMP_UPLOADS = 'temporary-files'
 SUB_FOLDER = 'etlJobHistory'
 
+_FORMAT_EXTENSIONS = {
+    "csv": ".csv",
+    "ndjson": ".ndjson",
+    "json": ".json",
+    "parquet": ".parquet",
+    "xml": ".xml",
+    "xlsx": ".xlsx",
+}
+_CONTAINER_EXTENSIONS = {
+    "zip": ".zip",
+    "gzip": ".gz",
+    "tar": ".tar",
+}
+
 
 class PipelineConductor:
     def __init__(self, 
@@ -159,9 +176,49 @@ class PipelineConductor:
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
+    def get_data_stream(self):
+        """
+        POC: pull a DataStream from the connector and upload raw bytes to S3.
+
+        Not wired into the default SOURCE path yet — call explicitly when testing.
+        Retains extract/normalize hints via a sibling .meta.json object.
+        """
+        if not hasattr(self.connector, "get_data_stream"):
+            raise errors.DataError(
+                "Connector does not implement get_data_stream; cannot use stream transfer."
+            )
+
+        self.log(self.log_templates.GET_DATA_START.format(
+            self.pipeline_details.source_object_id,
+            ", ".join(str(fid) for fid in self._get_field_ids())
+        ))
+
+        data_stream = self.connector.get_data_stream(
+            self.pipeline_details.source_object_id,
+            self._get_field_ids(),
+            filters=self._get_filters(),
+            options=self.pipeline_details.options,
+        )
+        if not isinstance(data_stream, DataStream):
+            raise errors.DataError(
+                "get_data_stream must return a dc_sdk.data_stream.DataStream instance."
+            )
+
+        key_name = self._process_data_stream(data_stream)
+
+        self.log(self.log_templates.GET_DATA_FINISH.format(
+            self.row_count, self.pipeline_details.source_object_id
+        ))
+        return key_name
+
     def load_data(self, batch_start):
         self._ensure_mapping_has_default_data_types()
-        self.log(self.log_templates.LOAD_DATA_START.format(self.pipeline_details.destination_object_id, len(self._get_field_ids())))
+        destination_object_id = resolve_destination_object_template(
+            self.pipeline_details.destination_object_id,
+            source_object_id=getattr(self.pipeline_details, "source_object_id", None),
+            source_connector_nm=getattr(self.pipeline_details, "source_connector_nm", None),
+        )
+        self.log(self.log_templates.LOAD_DATA_START.format(destination_object_id, len(self._get_field_ids())))
         
         keys = []
         if PipelineEnvironment.platform == "aws":
@@ -171,7 +228,7 @@ class PipelineConductor:
 
         if not keys:
             # Call load_data with empty data when there are no keys
-            loaded = self._call_connector_load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
+            loaded = self._call_connector_load_data([], destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
             if not loaded:
                 raise errors.LoadDataError("Loading data failed.")
         else:
@@ -184,8 +241,8 @@ class PipelineConductor:
 
                 
                 data = json.load(file_object)
-                self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), self.pipeline_details.destination_object_id))
-                loaded = self._call_connector_load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
+                self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), destination_object_id))
+                loaded = self._call_connector_load_data(data, destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
                 if loaded:
                     self.row_count += len(data)
                     if self.mode == "prod":
@@ -196,7 +253,7 @@ class PipelineConductor:
                         self.aws.delete_object(key)
                 else:
                     raise errors.LoadDataError("Loading data failed.")
-        self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, self.pipeline_details.destination_object_id))
+        self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, destination_object_id))
         self._persist_sync_cursor_after_load()
 
     def _create_connector(self, connector_cls, credentials):
@@ -422,6 +479,84 @@ class PipelineConductor:
         
         return limit_reached
 
+    def _process_data_stream(self, data_stream: DataStream):
+        """Upload a DataStream's bytes to object storage and persist normalize hints."""
+        self.batch_index += 1
+        extension = self._data_stream_extension(data_stream)
+
+        if self.mode == "prod":
+            key_name = f"e{self.pipeline_run_history_id}-b{self.batch_index}{extension}"
+        else:
+            key_name = f"devTransfers/e{self.prefix}-b{self.batch_index}{extension}"
+
+        if PipelineEnvironment.platform == "aws":
+            key_name = f"transfers/{key_name}"
+            self.aws.upload_stream(
+                key_name,
+                data_stream.stream,
+                content_type=data_stream.media_type,
+            )
+            meta_key = f"{key_name}.meta.json"
+            meta_buffer = io.StringIO(json.dumps(data_stream.hints()))
+            self.aws.upload_object(meta_key, json_buffer=meta_buffer)
+            self.bytes_transferred += self.aws.get_object_size(key_name)
+        elif PipelineEnvironment.platform == "azure":
+            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
+            if not hasattr(self.azure, "upload_stream"):
+                raise errors.DataError(
+                    "Azure upload_stream is not implemented for DataStream POC."
+                )
+            self.azure.upload_stream(
+                key_name,
+                data_stream.stream,
+                content_type=data_stream.media_type,
+            )
+            meta_key = f"{key_name}.meta.json"
+            meta_buffer = io.StringIO(json.dumps(data_stream.hints()))
+            self.azure.upload_object(meta_key, json_buffer=meta_buffer)
+            self.bytes_transferred += self.azure.get_object_size(key_name)
+        else:
+            raise errors.DataError(
+                f"Unsupported platform for DataStream upload: {PipelineEnvironment.platform}"
+            )
+
+        self.successful_keys.append(key_name)
+        self.internal_log(self.log_templates.INTERNAL_S3_UPLOAD_LOG.format(key_name))
+
+        row_count = data_stream.metadata.get("row_count")
+        if row_count is not None:
+            try:
+                self.row_count += int(row_count)
+            except (TypeError, ValueError):
+                pass
+
+        if self.mode == "prod":
+            self.update_history({
+                "updateAction": "ROWS_RETRIEVED",
+                "RowsRetrievedNBR": self.row_count,
+            })
+
+        return key_name
+
+    @staticmethod
+    def _data_stream_extension(data_stream: DataStream) -> str:
+        if data_stream.filename:
+            _, ext = os.path.splitext(data_stream.filename)
+            if ext:
+                return ext.lower()
+
+        if data_stream.container:
+            container_ext = _CONTAINER_EXTENSIONS.get(data_stream.container.lower())
+            if container_ext:
+                return container_ext
+
+        if data_stream.format:
+            format_ext = _FORMAT_EXTENSIONS.get(data_stream.format.lower())
+            if format_ext:
+                return format_ext
+
+        return ".bin"
+
     def _get_credentials(self):
         is_source = self.task == "SOURCE"
 

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_sdk/src/services/aws.py

@@ -1,7 +1,7 @@
 import os
 import boto3
 import json
-from typing import Dict
+from typing import Dict, Iterable, Optional
 from base64 import b64decode
 from Crypto.Cipher import AES
 
@@ -14,6 +14,38 @@ s3_resource = boto3.resource('s3')
 ecs_resource = boto3.client('ecs')
 kms_client = boto3.client('kms')
 
+
+class _ChunkIteratorReader:
+    """File-like wrapper so boto3 can upload an Iterable[bytes] via upload_fileobj."""
+
+    def __init__(self, chunks: Iterable[bytes]):
+        self._iterator = iter(chunks)
+        self._buffer = b""
+        self._exhausted = False
+
+    def read(self, size: int = -1) -> bytes:
+        if size == 0:
+            return b""
+
+        if size < 0:
+            parts = [self._buffer]
+            self._buffer = b""
+            if not self._exhausted:
+                parts.extend(self._iterator)
+                self._exhausted = True
+            return b"".join(parts)
+
+        while len(self._buffer) < size and not self._exhausted:
+            try:
+                self._buffer += next(self._iterator)
+            except StopIteration:
+                self._exhausted = True
+
+        out = self._buffer[:size]
+        self._buffer = self._buffer[size:]
+        return out
+
+
 class AwsService:
     def __init__(self, s3_bucket) -> None:
         self.s3_bucket = s3_bucket
@@ -48,6 +80,25 @@ class AwsService:
         else:
             raise ValueError("Either json_buffer or file_path must be provided")
 
+    def upload_stream(
+        self,
+        key_name: str,
+        stream: Iterable[bytes],
+        content_type: Optional[str] = None,
+    ):
+        """Upload an iterable of byte chunks to S3 without requiring a local file."""
+        extra_args = {}
+        if content_type:
+            extra_args["ContentType"] = content_type
+
+        reader = _ChunkIteratorReader(stream)
+        if extra_args:
+            s3_client.upload_fileobj(
+                reader, self.s3_bucket, key_name, ExtraArgs=extra_args
+            )
+        else:
+            s3_client.upload_fileobj(reader, self.s3_bucket, key_name)
+
     def get_object_size(self, key_name):
         response = s3_client.head_object(
             Bucket=self.s3_bucket, Key=key_name)

{dc_python_sdk-1.5.48 → dc_python_sdk-1.5.50}/src/dc_sdk/types.py

@@ -1,5 +1,7 @@
 from typing import Protocol, Any, Dict, List, Optional
 
+from dc_sdk.data_stream import DataStream
+
 
 class ConnectorProtocol(Protocol):
 
@@ -25,6 +27,14 @@ class ConnectorProtocol(Protocol):
         options: Dict[str, Any] = {}
     ) -> Any: ...
 
+    def get_data_stream(
+        self,
+        object_id: str,
+        field_ids: List[str],
+        filters: Optional[Dict[str, Any]] = None,
+        options: Dict[str, Any] = {}
+    ) -> DataStream: ...
+
     def load_data(
         self,
         data: List[Dict[str, Any]],
@@ -35,4 +45,4 @@ class ConnectorProtocol(Protocol):
         total_batches: int
     ) -> Any: ...
 
-    def close(self) -> None: ...
+    def close(self) -> None: ...

dc_python_sdk-1.5.48/src/dc_sdk/src/mapping.py

@@ -1,109 +0,0 @@
-from dc_sdk import errors
-import logging
-from dc_sdk.src.services.loader import load_connector
-
-logger = logging.getLogger(__name__)
-class Mapping():
-    def __init__(self, credentials):
-        Connector = load_connector()
-        self.connector = Connector(credentials)
-
-    def authenticate(self):
-        results = None
-        metadata = None
-        message = None
-
-        if self.connector.authenticate():
-            try:
-                metadata = self.connector.get_metadata()
-            except errors.NotImplementedError:
-                pass
-            except:
-                logger.exception("Error getting metadata")
-
-            results = {
-                'metadata': metadata
-            }
-
-            message = "Authenticated successfully"
-        else:
-            raise errors.AuthenticationError(message="Failed to authenticate")
-
-        return [results, message]
-
-    def connect_get_objects(self):
-        results = None
-        metadata = None
-        objects = None
-        message = None
-
-        if self.connector.authenticate():
-            objects = self.connector.get_objects()
-            try:
-                metadata = self.connector.get_metadata()
-            except errors.NotImplementedError:
-                pass
-            except Exception as e:
-                print("Error getting metadata")
-                print(str(e))
-
-            results = {
-                'metadata': metadata,
-                'objects': objects,
-            }
-            message = "Authenticated successfully"
-        else:
-            # If not authenticated, raise the authentication error.
-            raise errors.AuthenticationError(message="Failed to authenticate")
-
-        return [results, message]
-        
-    def test_connection(self):
-        results = None
-        message = None
-
-        if self.connector.authenticate():
-            message = "Authenticated successfully"
-        else:
-            # If not authenticated, raise the authentication error.
-            raise errors.AuthenticationError(message="Failed to authenticate")
-
-        return [results, message]
-
-    def get_fields(self, object_id, options):
-        results = None
-        message = None
-        if self.connector.authenticate():
-            results = self.connector.get_fields(object_id, options=options)
-            message = "Retrieved fields"
-        else:
-            # If not authenticated, raise the authentication error.
-            raise errors.AuthenticationError("Failed to authenticate")
-
-        return [results, message]
-
-    def get_five_row_preview(self, object_id, field_ids, options, n_rows = 5, filters = None, next_page = None):
-        results = None
-        message = None
-
-        if self.connector.authenticate():
-            results = self.connector.get_data(object_id, field_ids=field_ids, n_rows=n_rows, filters=filters, next_page=next_page, options=options)
-            message = "Retrieved 5 row preview"
-        else:
-            # If not authenticated, raise the authentication error.
-            raise errors.AuthenticationError("Failed to authenticate")
-
-        return [results, message]
-
-    def get_data(self, object_id, field_ids, n_rows, next_page, options):
-        results = None
-        message = None
-
-        if self.connector.authenticate():
-            results = self.connector.get_data(object_id, field_ids=field_ids, n_rows=n_rows, next_page=next_page, options=options)
-            message = "Retrieved data"
-        else:
-            # If not authenticated, raise the authentication error.
-            raise errors.AuthenticationError("Failed to authenticate")
-
-        return [results, message]