dc-python-sdk 1.5.49__tar.gz → 1.5.51__tar.gz

{dc_python_sdk-1.5.49/src/dc_python_sdk.egg-info → dc_python_sdk-1.5.51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.49
+Version: 1.5.51
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.49"
+version = "1.5.51"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.49
+version = 1.5.51
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.49
+Version: 1.5.51
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/src/dc_python_sdk.egg-info/SOURCES.txt

@@ -11,6 +11,7 @@ src/dc_python_sdk.egg-info/top_level.txt
 src/dc_sdk/__init__.py
 src/dc_sdk/app.py
 src/dc_sdk/cli.py
+src/dc_sdk/data_stream.py
 src/dc_sdk/errors.py
 src/dc_sdk/handler.py
 src/dc_sdk/types.py
@@ -18,6 +19,7 @@ src/dc_sdk/src/__init__.py
 src/dc_sdk/src/ai.py
 src/dc_sdk/src/ai_http.py
 src/dc_sdk/src/connection_status.py
+src/dc_sdk/src/destination_object_template.py
 src/dc_sdk/src/mapping.py
 src/dc_sdk/src/pipeline.py
 src/dc_sdk/src/server.py

dc_python_sdk-1.5.51/src/dc_sdk/__init__.py

@@ -0,0 +1,3 @@
+from dc_sdk.data_stream import DataStream
+
+__all__ = ["DataStream"]

dc_python_sdk-1.5.51/src/dc_sdk/data_stream.py

@@ -0,0 +1,33 @@
+from dataclasses import dataclass, field
+from typing import Any, Dict, Iterable, Optional
+
+
+@dataclass
+class DataStream:
+    """Byte stream returned by connector get_data_stream for transfer/upload."""
+
+    stream: Iterable[bytes]
+
+    # What are the bytes?
+    media_type: str
+
+    # Optional processing hints
+    container: Optional[str] = None  # zip, gzip, tar
+    format: Optional[str] = None  # csv, ndjson, parquet, xml, xlsx
+
+    # Optional metadata
+    filename: Optional[str] = None
+    encoding: Optional[str] = None
+
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def hints(self) -> Dict[str, Any]:
+        """Serializable extract/normalize hints (excludes the live stream)."""
+        return {
+            "media_type": self.media_type,
+            "container": self.container,
+            "format": self.format,
+            "filename": self.filename,
+            "encoding": self.encoding,
+            "metadata": self.metadata,
+        }

dc_python_sdk-1.5.51/src/dc_sdk/src/destination_object_template.py

@@ -0,0 +1,74 @@
+"""Resolve dynamic tokens in destination object paths / file names at run time."""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Optional
+
+_TOKEN_RE = re.compile(r"\{([A-Za-z0-9_-]+)\}")
+
+_DATE_FORMATS = {
+    "YYYYMMDD": "%Y%m%d",
+    "YYYY-MM-DD": "%Y-%m-%d",
+    "YYMMDD": "%y%m%d",
+    "HHMMSS": "%H%M%S",
+    "YYYYMMDDHHMMSS": "%Y%m%d%H%M%S",
+}
+
+
+def _slug(value: Optional[str]) -> str:
+    if value is None:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    text = re.sub(r"[^A-Za-z0-9]+", "_", text)
+    return text.strip("_")
+
+
+def resolve_destination_object_template(
+    template: Optional[str],
+    *,
+    source_object_id: Optional[str] = None,
+    source_connector_nm: Optional[str] = None,
+    when: Optional[datetime] = None,
+) -> str:
+    """
+    Expand `{YYYYMMDD}`, `{object}`, `{connector}`, etc. in a destination path.
+
+    Unknown tokens are left unchanged so mis-typed braces remain visible in logs.
+    """
+    if template is None:
+        return ""
+    value = str(template)
+    if "{" not in value:
+        return value
+
+    now = when or datetime.now(timezone.utc)
+    if now.tzinfo is None:
+        now = now.replace(tzinfo=timezone.utc)
+
+    replacements = {
+        "object": _slug(source_object_id),
+        "connector": _slug(source_connector_nm),
+    }
+    for token, fmt in _DATE_FORMATS.items():
+        replacements[token] = now.strftime(fmt)
+
+    def _replace(match: re.Match) -> str:
+        key = match.group(1)
+        if key in replacements:
+            return replacements[key]
+        # Autocomplete may insert uppercase tokens; users often type lowercase
+        # (`{hhmmss}`). Resolve date formats case-insensitively, and
+        # `{object}` / `{connector}` case-insensitively.
+        upper = key.upper()
+        if upper in _DATE_FORMATS:
+            return replacements[upper]
+        lower = key.lower()
+        if lower in ("object", "connector"):
+            return replacements[lower]
+        return match.group(0)
+
+    return _TOKEN_RE.sub(_replace, value)

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/src/dc_sdk/src/mapping.py

@@ -71,12 +71,21 @@ class Mapping():
 
         self._ensure_session(force_authenticate=False)
 
-    def _safe_get_metadata(self):
+    def _get_metadata(self, required=False):
+        """
+        Fetch connector metadata.
+
+        On connect/authenticate paths, failures must surface (required=True) so
+        report-catalog connectors cannot look Connected with empty accounts.
+        Soft-fail (required=False) only for optional metadata enrichment.
+        """
         try:
             return self.connector.get_metadata()
         except errors.NotImplementedError:
             return None
         except Exception:
+            if required:
+                raise
             logger.exception("Error getting metadata")
             return None
 
@@ -86,7 +95,7 @@ class Mapping():
 
         self._ensure_session(force_authenticate=force_authenticate)
 
-        metadata = self._safe_get_metadata()
+        metadata = self._get_metadata(required=True)
         results = {
             "metadata": metadata
         }
@@ -106,7 +115,7 @@ class Mapping():
         )
 
         objects = self.connector.get_objects()
-        metadata = self._safe_get_metadata()
+        metadata = self._get_metadata(required=True)
 
         results = {
             "metadata": metadata,
@@ -122,7 +131,7 @@ class Mapping():
             force_authenticate=force_authenticate,
         )
 
-        metadata = self._safe_get_metadata()
+        metadata = self._get_metadata(required=True)
         return [{"metadata": metadata}, "Retrieved metadata"]
 
     def get_objects_only(self, skip_authenticate=False, force_authenticate=False, include_metadata=True):
@@ -135,7 +144,8 @@ class Mapping():
         results = {"objects": objects}
 
         if include_metadata:
-            results["metadata"] = self._safe_get_metadata()
+            # Optional enrichment on object-only fetches; connect path uses required=True.
+            results["metadata"] = self._get_metadata(required=False)
 
         return [results, "Retrieved objects"]
 

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/src/dc_sdk/src/pipeline.py

@@ -1,4 +1,5 @@
 import json, io, math, re, inspect
+import os
 import time
 from .services.environment import PipelineEnvironment
 from .services.api import DataConnectorAPI
@@ -6,10 +7,26 @@ from .models.log_templates import LogTemplates
 from .services.aws import AwsService
 from .models import errors
 from .services.loader import load_connector
+from .destination_object_template import resolve_destination_object_template
+from dc_sdk.data_stream import DataStream
 
 TEMP_UPLOADS = 'temporary-files'
 SUB_FOLDER = 'etlJobHistory'
 
+_FORMAT_EXTENSIONS = {
+    "csv": ".csv",
+    "ndjson": ".ndjson",
+    "json": ".json",
+    "parquet": ".parquet",
+    "xml": ".xml",
+    "xlsx": ".xlsx",
+}
+_CONTAINER_EXTENSIONS = {
+    "zip": ".zip",
+    "gzip": ".gz",
+    "tar": ".tar",
+}
+
 
 class PipelineConductor:
     def __init__(self, 
@@ -159,9 +176,49 @@ class PipelineConductor:
 
         self.log(self.log_templates.GET_DATA_FINISH.format(self.row_count, self.pipeline_details.source_object_id))
 
+    def get_data_stream(self):
+        """
+        POC: pull a DataStream from the connector and upload raw bytes to S3.
+
+        Not wired into the default SOURCE path yet — call explicitly when testing.
+        Retains extract/normalize hints via a sibling .meta.json object.
+        """
+        if not hasattr(self.connector, "get_data_stream"):
+            raise errors.DataError(
+                "Connector does not implement get_data_stream; cannot use stream transfer."
+            )
+
+        self.log(self.log_templates.GET_DATA_START.format(
+            self.pipeline_details.source_object_id,
+            ", ".join(str(fid) for fid in self._get_field_ids())
+        ))
+
+        data_stream = self.connector.get_data_stream(
+            self.pipeline_details.source_object_id,
+            self._get_field_ids(),
+            filters=self._get_filters(),
+            options=self.pipeline_details.options,
+        )
+        if not isinstance(data_stream, DataStream):
+            raise errors.DataError(
+                "get_data_stream must return a dc_sdk.data_stream.DataStream instance."
+            )
+
+        key_name = self._process_data_stream(data_stream)
+
+        self.log(self.log_templates.GET_DATA_FINISH.format(
+            self.row_count, self.pipeline_details.source_object_id
+        ))
+        return key_name
+
     def load_data(self, batch_start):
         self._ensure_mapping_has_default_data_types()
-        self.log(self.log_templates.LOAD_DATA_START.format(self.pipeline_details.destination_object_id, len(self._get_field_ids())))
+        destination_object_id = resolve_destination_object_template(
+            self.pipeline_details.destination_object_id,
+            source_object_id=getattr(self.pipeline_details, "source_object_id", None),
+            source_connector_nm=getattr(self.pipeline_details, "source_connector_nm", None),
+        )
+        self.log(self.log_templates.LOAD_DATA_START.format(destination_object_id, len(self._get_field_ids())))
         
         keys = []
         if PipelineEnvironment.platform == "aws":
@@ -171,7 +228,7 @@ class PipelineConductor:
 
         if not keys:
             # Call load_data with empty data when there are no keys
-            loaded = self._call_connector_load_data([], self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
+            loaded = self._call_connector_load_data([], destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, 0, 1)
             if not loaded:
                 raise errors.LoadDataError("Loading data failed.")
         else:
@@ -184,8 +241,8 @@ class PipelineConductor:
 
                 
                 data = json.load(file_object)
-                self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), self.pipeline_details.destination_object_id))
-                loaded = self._call_connector_load_data(data, self.pipeline_details.destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
+                self.log(self.log_templates.LOAD_DATA_LOADED.format(len(data), destination_object_id))
+                loaded = self._call_connector_load_data(data, destination_object_id, self._get_mapping(), self.pipeline_details.update_method_cd, index, len(keys))
                 if loaded:
                     self.row_count += len(data)
                     if self.mode == "prod":
@@ -196,7 +253,7 @@ class PipelineConductor:
                         self.aws.delete_object(key)
                 else:
                     raise errors.LoadDataError("Loading data failed.")
-        self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, self.pipeline_details.destination_object_id))
+        self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, destination_object_id))
         self._persist_sync_cursor_after_load()
 
     def _create_connector(self, connector_cls, credentials):
@@ -422,6 +479,84 @@ class PipelineConductor:
         
         return limit_reached
 
+    def _process_data_stream(self, data_stream: DataStream):
+        """Upload a DataStream's bytes to object storage and persist normalize hints."""
+        self.batch_index += 1
+        extension = self._data_stream_extension(data_stream)
+
+        if self.mode == "prod":
+            key_name = f"e{self.pipeline_run_history_id}-b{self.batch_index}{extension}"
+        else:
+            key_name = f"devTransfers/e{self.prefix}-b{self.batch_index}{extension}"
+
+        if PipelineEnvironment.platform == "aws":
+            key_name = f"transfers/{key_name}"
+            self.aws.upload_stream(
+                key_name,
+                data_stream.stream,
+                content_type=data_stream.media_type,
+            )
+            meta_key = f"{key_name}.meta.json"
+            meta_buffer = io.StringIO(json.dumps(data_stream.hints()))
+            self.aws.upload_object(meta_key, json_buffer=meta_buffer)
+            self.bytes_transferred += self.aws.get_object_size(key_name)
+        elif PipelineEnvironment.platform == "azure":
+            key_name = f"{PipelineEnvironment.app_env}/{key_name}"
+            if not hasattr(self.azure, "upload_stream"):
+                raise errors.DataError(
+                    "Azure upload_stream is not implemented for DataStream POC."
+                )
+            self.azure.upload_stream(
+                key_name,
+                data_stream.stream,
+                content_type=data_stream.media_type,
+            )
+            meta_key = f"{key_name}.meta.json"
+            meta_buffer = io.StringIO(json.dumps(data_stream.hints()))
+            self.azure.upload_object(meta_key, json_buffer=meta_buffer)
+            self.bytes_transferred += self.azure.get_object_size(key_name)
+        else:
+            raise errors.DataError(
+                f"Unsupported platform for DataStream upload: {PipelineEnvironment.platform}"
+            )
+
+        self.successful_keys.append(key_name)
+        self.internal_log(self.log_templates.INTERNAL_S3_UPLOAD_LOG.format(key_name))
+
+        row_count = data_stream.metadata.get("row_count")
+        if row_count is not None:
+            try:
+                self.row_count += int(row_count)
+            except (TypeError, ValueError):
+                pass
+
+        if self.mode == "prod":
+            self.update_history({
+                "updateAction": "ROWS_RETRIEVED",
+                "RowsRetrievedNBR": self.row_count,
+            })
+
+        return key_name
+
+    @staticmethod
+    def _data_stream_extension(data_stream: DataStream) -> str:
+        if data_stream.filename:
+            _, ext = os.path.splitext(data_stream.filename)
+            if ext:
+                return ext.lower()
+
+        if data_stream.container:
+            container_ext = _CONTAINER_EXTENSIONS.get(data_stream.container.lower())
+            if container_ext:
+                return container_ext
+
+        if data_stream.format:
+            format_ext = _FORMAT_EXTENSIONS.get(data_stream.format.lower())
+            if format_ext:
+                return format_ext
+
+        return ".bin"
+
     def _get_credentials(self):
         is_source = self.task == "SOURCE"
 

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/src/dc_sdk/src/services/aws.py

@@ -1,7 +1,7 @@
 import os
 import boto3
 import json
-from typing import Dict
+from typing import Dict, Iterable, Optional
 from base64 import b64decode
 from Crypto.Cipher import AES
 
@@ -14,6 +14,38 @@ s3_resource = boto3.resource('s3')
 ecs_resource = boto3.client('ecs')
 kms_client = boto3.client('kms')
 
+
+class _ChunkIteratorReader:
+    """File-like wrapper so boto3 can upload an Iterable[bytes] via upload_fileobj."""
+
+    def __init__(self, chunks: Iterable[bytes]):
+        self._iterator = iter(chunks)
+        self._buffer = b""
+        self._exhausted = False
+
+    def read(self, size: int = -1) -> bytes:
+        if size == 0:
+            return b""
+
+        if size < 0:
+            parts = [self._buffer]
+            self._buffer = b""
+            if not self._exhausted:
+                parts.extend(self._iterator)
+                self._exhausted = True
+            return b"".join(parts)
+
+        while len(self._buffer) < size and not self._exhausted:
+            try:
+                self._buffer += next(self._iterator)
+            except StopIteration:
+                self._exhausted = True
+
+        out = self._buffer[:size]
+        self._buffer = self._buffer[size:]
+        return out
+
+
 class AwsService:
     def __init__(self, s3_bucket) -> None:
         self.s3_bucket = s3_bucket
@@ -48,6 +80,25 @@ class AwsService:
         else:
             raise ValueError("Either json_buffer or file_path must be provided")
 
+    def upload_stream(
+        self,
+        key_name: str,
+        stream: Iterable[bytes],
+        content_type: Optional[str] = None,
+    ):
+        """Upload an iterable of byte chunks to S3 without requiring a local file."""
+        extra_args = {}
+        if content_type:
+            extra_args["ContentType"] = content_type
+
+        reader = _ChunkIteratorReader(stream)
+        if extra_args:
+            s3_client.upload_fileobj(
+                reader, self.s3_bucket, key_name, ExtraArgs=extra_args
+            )
+        else:
+            s3_client.upload_fileobj(reader, self.s3_bucket, key_name)
+
     def get_object_size(self, key_name):
         response = s3_client.head_object(
             Bucket=self.s3_bucket, Key=key_name)

{dc_python_sdk-1.5.49 → dc_python_sdk-1.5.51}/src/dc_sdk/types.py

@@ -1,5 +1,7 @@
 from typing import Protocol, Any, Dict, List, Optional
 
+from dc_sdk.data_stream import DataStream
+
 
 class ConnectorProtocol(Protocol):
 
@@ -25,6 +27,14 @@ class ConnectorProtocol(Protocol):
         options: Dict[str, Any] = {}
     ) -> Any: ...
 
+    def get_data_stream(
+        self,
+        object_id: str,
+        field_ids: List[str],
+        filters: Optional[Dict[str, Any]] = None,
+        options: Dict[str, Any] = {}
+    ) -> DataStream: ...
+
     def load_data(
         self,
         data: List[Dict[str, Any]],
@@ -35,4 +45,4 @@ class ConnectorProtocol(Protocol):
         total_batches: int
     ) -> Any: ...
 
-    def close(self) -> None: ...
+    def close(self) -> None: ...