dc-python-sdk 1.5.54__tar.gz → 1.6.1__tar.gz

{dc_python_sdk-1.5.54/src/dc_python_sdk.egg-info → dc_python_sdk-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.54
+Version: 1.6.1
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector
@@ -14,7 +14,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastapi
 Requires-Dist: uvicorn
-Requires-Dist: awslambdaric
+Requires-Dist: awslambdaric==4.0.4
 Requires-Dist: requests
 Requires-Dist: boto3>=1.40.0
 Requires-Dist: openai
@@ -94,6 +94,41 @@ def get_available_objects():
     return objects
 ```
 
+### Selective discovery in the Lambda handler
+
+Authenticate with action `3`, then request only the metadata needed by the current
+screen with action `5`:
+
+```json
+{"action": 5, "sections": ["accounts"], "credentials": {}}
+```
+
+Connectors opt in by defining `get_metadata(self, sections=None)`. Fabric supports
+`accounts`, `schemas`, and `folders`; callers can request more than one section.
+Requested sections retain their usual keys in the metadata dictionary, and static
+capability fields may also be returned. Unrequested dynamic keys are omitted,
+not returned as empty lists. Callers should merge partial metadata into existing
+metadata for the same account, and clear cached metadata when switching accounts.
+
+Action `6` accepts the same optional parameter for `get_objects(self, sections=None)`;
+Fabric supports `tables` and `files`. Use `include_metadata: false` to avoid an
+additional full metadata request when fetching only objects. Object results remain
+a list in the existing response envelope.
+
+Omitting `sections` (or using null) retains full discovery. An empty list requests
+no dynamic sections on an opted-in connector. The SDK validates a list of non-empty
+strings and passes it only to methods with an explicit `sections` keyword; legacy
+connectors still perform full discovery with no arguments. Each opted-in connector
+validates its supported section names. Internal connector TypeErrors are propagated,
+never retried as a legacy call. The SDK does not filter a legacy response or claim
+that a legacy connector avoided fetching unrequested data.
+
+The SDK returns `error_phase: "metadata"` for discovery errors and keeps successful
+authentication status. `AuthenticationError` still invalidates authentication.
+API and UI consumers must use this distinction to show a discovery retry instead
+of requiring reconnection. This contract applies to the Lambda handler; the local
+HTTP server below uses a separate method-based request format.
+
 ### Running the local HTTP server
 
 Install the SDK and start the HTTP server that wraps your connector:

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/README.md

@@ -61,6 +61,41 @@ def get_available_objects():
     return objects
 ```
 
+### Selective discovery in the Lambda handler
+
+Authenticate with action `3`, then request only the metadata needed by the current
+screen with action `5`:
+
+```json
+{"action": 5, "sections": ["accounts"], "credentials": {}}
+```
+
+Connectors opt in by defining `get_metadata(self, sections=None)`. Fabric supports
+`accounts`, `schemas`, and `folders`; callers can request more than one section.
+Requested sections retain their usual keys in the metadata dictionary, and static
+capability fields may also be returned. Unrequested dynamic keys are omitted,
+not returned as empty lists. Callers should merge partial metadata into existing
+metadata for the same account, and clear cached metadata when switching accounts.
+
+Action `6` accepts the same optional parameter for `get_objects(self, sections=None)`;
+Fabric supports `tables` and `files`. Use `include_metadata: false` to avoid an
+additional full metadata request when fetching only objects. Object results remain
+a list in the existing response envelope.
+
+Omitting `sections` (or using null) retains full discovery. An empty list requests
+no dynamic sections on an opted-in connector. The SDK validates a list of non-empty
+strings and passes it only to methods with an explicit `sections` keyword; legacy
+connectors still perform full discovery with no arguments. Each opted-in connector
+validates its supported section names. Internal connector TypeErrors are propagated,
+never retried as a legacy call. The SDK does not filter a legacy response or claim
+that a legacy connector avoided fetching unrequested data.
+
+The SDK returns `error_phase: "metadata"` for discovery errors and keeps successful
+authentication status. `AuthenticationError` still invalidates authentication.
+API and UI consumers must use this distinction to show a discovery retry instead
+of requiring reconnection. This contract applies to the Lambda handler; the local
+HTTP server below uses a separate method-based request format.
+
 ### Running the local HTTP server
 
 Install the SDK and start the HTTP server that wraps your connector:

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dc-python-sdk"
-version = "1.5.54"
+version = "1.6.1"
 description = "Data Connector Python SDK"
 readme = "README.md"
 requires-python = ">=3.6"
@@ -19,7 +19,7 @@ classifiers = [
 dependencies = [
     "fastapi",
     "uvicorn",
-    "awslambdaric",
+    "awslambdaric==4.0.4",
     "requests",
     "boto3>=1.40.0",
     "openai",

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = dc-python-sdk
-version = 1.5.54
+version = 1.6.1
 author = DataConnector
 author_email = josh@dataconnector.com
 description = A small example package
@@ -22,7 +22,7 @@ python_requires = >=3.6
 install_requires = 
 	fastapi
 	uvicorn
-	awslambdaric
+	awslambdaric==4.0.4
 	pycryptodome
 	requests
 	boto3>=1.40.0

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1/src/dc_python_sdk.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dc-python-sdk
-Version: 1.5.54
+Version: 1.6.1
 Summary: Data Connector Python SDK
 Home-page: https://github.com/data-connector/dc-python-sdk
 Author: DataConnector
@@ -14,7 +14,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastapi
 Requires-Dist: uvicorn
-Requires-Dist: awslambdaric
+Requires-Dist: awslambdaric==4.0.4
 Requires-Dist: requests
 Requires-Dist: boto3>=1.40.0
 Requires-Dist: openai
@@ -94,6 +94,41 @@ def get_available_objects():
     return objects
 ```
 
+### Selective discovery in the Lambda handler
+
+Authenticate with action `3`, then request only the metadata needed by the current
+screen with action `5`:
+
+```json
+{"action": 5, "sections": ["accounts"], "credentials": {}}
+```
+
+Connectors opt in by defining `get_metadata(self, sections=None)`. Fabric supports
+`accounts`, `schemas`, and `folders`; callers can request more than one section.
+Requested sections retain their usual keys in the metadata dictionary, and static
+capability fields may also be returned. Unrequested dynamic keys are omitted,
+not returned as empty lists. Callers should merge partial metadata into existing
+metadata for the same account, and clear cached metadata when switching accounts.
+
+Action `6` accepts the same optional parameter for `get_objects(self, sections=None)`;
+Fabric supports `tables` and `files`. Use `include_metadata: false` to avoid an
+additional full metadata request when fetching only objects. Object results remain
+a list in the existing response envelope.
+
+Omitting `sections` (or using null) retains full discovery. An empty list requests
+no dynamic sections on an opted-in connector. The SDK validates a list of non-empty
+strings and passes it only to methods with an explicit `sections` keyword; legacy
+connectors still perform full discovery with no arguments. Each opted-in connector
+validates its supported section names. Internal connector TypeErrors are propagated,
+never retried as a legacy call. The SDK does not filter a legacy response or claim
+that a legacy connector avoided fetching unrequested data.
+
+The SDK returns `error_phase: "metadata"` for discovery errors and keeps successful
+authentication status. `AuthenticationError` still invalidates authentication.
+API and UI consumers must use this distinction to show a discovery retry instead
+of requiring reconnection. This contract applies to the Lambda handler; the local
+HTTP server below uses a separate method-based request format.
+
 ### Running the local HTTP server
 
 Install the SDK and start the HTTP server that wraps your connector:

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_python_sdk.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ src/dc_sdk/errors.py
 src/dc_sdk/file_utils.py
 src/dc_sdk/handler.py
 src/dc_sdk/json_safe.py
+src/dc_sdk/row_filters.py
 src/dc_sdk/types.py
 src/dc_sdk/src/__init__.py
 src/dc_sdk/src/ai.py

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_python_sdk.egg-info/requires.txt

@@ -1,6 +1,6 @@
 fastapi
 uvicorn
-awslambdaric
+awslambdaric==4.0.4
 requests
 boto3>=1.40.0
 openai

dc_python_sdk-1.6.1/src/dc_sdk/handler.py

@@ -0,0 +1,122 @@
+from dc_sdk.row_filters import apply_data_filters, compile_sync_filters
+from dc_sdk.errors import Error, AuthenticationError
+from dc_sdk.json_safe import to_json_safe
+from dc_sdk.src.mapping import Mapping
+from dc_sdk.src.connection_status import STATUS_ERROR
+import traceback
+from importlib.metadata import version
+
+def get_action_name(action: int) -> str:
+    return {
+        0: "authenticating",
+        1: "retrieving fields",
+        2: "retrieving 5 row preview",
+        3: "testing connection",
+        4: "starting the connector",
+        5: "retrieving metadata",
+        6: "retrieving objects",
+    }[action]
+
+def handler(event, context):
+    print("version: ", version("dc-python-sdk"))
+    """Lambda Handler"""
+    action_name = None
+    internal_error = False
+    message = None
+    results = None
+    error = None
+    error_phase = None
+    mapping = None
+    action = int(event['action']) if 'action' in event else 4
+    get_objects = event.get('get_objects', True)
+    skip_authenticate = event.get('skip_authenticate', False)
+    force_authenticate = event.get('force_authenticate', False)
+    include_metadata = event.get('include_metadata', True)
+    credentials_dict = event['credentials'] if 'credentials' in event else None
+    object_id = event['object_id'] if 'object_id' in event else None
+    field_ids = event['mapping'] if 'mapping' in event else None
+    options = event['options'] if 'options' in event else dict()
+    next_page = event['next_page'] if 'next_page' in event else None
+    n_rows = event['n_rows'] if 'n_rows' in event else None
+    filters = event['filters'] if 'filters' in event else None
+    data_filters = event['data_filters'] if 'data_filters' in event else None
+
+    try:
+        action_name = get_action_name(action)
+        mapping = Mapping(credentials_dict)
+
+        if action == 0:
+            if get_objects:
+                results, message = mapping.connect_get_objects(
+                    skip_authenticate=skip_authenticate,
+                    force_authenticate=force_authenticate,
+                    include_metadata=include_metadata,
+                )
+            else:
+                results, message = mapping.authenticate(
+                    force_authenticate=force_authenticate or not skip_authenticate,
+                    include_metadata=include_metadata,
+                )
+        elif action == 1:
+            results, message = mapping.get_fields(object_id, options)
+        elif action == 2:
+            results, message = mapping.get_five_row_preview(
+                object_id, field_ids, options, n_rows, filters, next_page)
+            # During rollout callers send both formats. Apply the canonical one
+            # once, so legacy date comparisons cannot remove valid whole-day matches.
+            preview_filters = (
+                compile_sync_filters(options['sync_filters'])
+                if isinstance(options, dict) and 'sync_filters' in options
+                else data_filters
+            )
+            results = apply_data_filters(results, preview_filters)
+        elif action == 3:
+            results, message = mapping.test_connection()
+        elif action == 5:
+            results, message = mapping.get_metadata_only(
+                skip_authenticate=skip_authenticate,
+                force_authenticate=force_authenticate,
+                sections=event.get('sections'),
+            )
+        elif action == 6:
+            results, message = mapping.get_objects_only(
+                skip_authenticate=skip_authenticate,
+                force_authenticate=force_authenticate,
+                include_metadata=include_metadata,
+                sections=event.get('sections'),
+            )
+        else:
+            raise Error("Invalid action", "InvalidActionError")
+    except Error as e:
+        error_phase = "authentication" if isinstance(e, AuthenticationError) else getattr(mapping, "request_phase", None)
+        message = e.message or str(e)
+        internal_error = e.internal
+        error_trace = traceback.format_exc()
+        error = error_trace
+        if mapping and (isinstance(e, AuthenticationError) or getattr(mapping, "request_phase", None) != "metadata"):
+            mapping._set_connection_status(STATUS_ERROR)
+
+    except Exception as e:
+        error_phase = getattr(mapping, "request_phase", None)
+        error_trace = traceback.format_exc()
+        error = error_trace
+        internal_error = True
+        if mapping and getattr(mapping, "request_phase", None) != "metadata":
+            mapping._set_connection_status(STATUS_ERROR)
+
+    if error:
+        print(error)
+
+    last_status_dsc = None
+    if mapping and mapping.connector.credentials:
+        last_status_dsc = mapping.connector.credentials.get("last_status_dsc")
+
+    return {
+        'message': message if not internal_error else f"Something went wrong with {action_name}",
+        'results': to_json_safe(results),
+        'credentials': mapping.connector.credentials if mapping else None,
+        'last_status_dsc': last_status_dsc,
+        'error': error,
+        'internal_error': internal_error,
+        'error_phase': error_phase,
+    }

dc_python_sdk-1.6.1/src/dc_sdk/row_filters.py

@@ -0,0 +1,246 @@
+from datetime import date, datetime, timedelta, timezone
+from zoneinfo import ZoneInfo
+
+
+def _calendar_date(value, zone):
+    if isinstance(value, datetime):
+        return value.astimezone(zone).date() if value.tzinfo else value.date()
+    if isinstance(value, date):
+        return value
+    text = str(value).strip()
+    # Date-only values represent calendar dates, not UTC midnight.
+    if len(text) == 10:
+        return date.fromisoformat(text)
+    parsed = datetime.fromisoformat(text.replace('Z', '+00:00'))
+    return parsed.astimezone(zone).date() if parsed.tzinfo else parsed.date()
+
+
+def compile_sync_filters(config, now=None):
+    """Resolve relative dates once per run; date boundaries include entire days."""
+    if not config:
+        return []
+    zone_name = config.get('timeZone') or 'UTC'
+    zone = ZoneInfo(zone_name)
+    today = (now or datetime.now(timezone.utc)).astimezone(zone).date()
+
+    def resolve(value):
+        offsets = {'__dq:today': 0, '__dq:yesterday': -1, '__dq:tomorrow': 1}
+        if value in offsets:
+            return today + timedelta(days=offsets[value])
+        return _calendar_date(value, zone)
+
+    supported = {
+        'is_equal', 'is_not_equal', 'is_empty', 'is_not_empty',
+        'text_contains', 'text_not_contains', 'text_starts_with', 'text_ends_with',
+        'gt', 'gte', 'lt', 'lte', 'between', 'not_between',
+        'date_is', 'date_before', 'date_after',
+    }
+    result = []
+    for column, entry in (config.get('values') or {}).items():
+        operator = entry.get('operator', 'none')
+        if operator == 'none':
+            continue
+        if operator not in supported:
+            raise ValueError('Unsupported sync filter operator.')
+        is_date = entry.get('kind') == 'date' or operator.startswith('date_')
+        value, value2 = entry.get('value'), entry.get('value2')
+        if operator not in ('is_empty', 'is_not_empty'):
+            if value is None or str(value).strip() == '':
+                raise ValueError('A sync filter value is required.')
+            if is_date:
+                value = resolve(value).isoformat()
+        if operator in ('between', 'not_between'):
+            if value2 is None or str(value2).strip() == '':
+                raise ValueError('Both sync filter range values are required.')
+            if is_date:
+                value2 = resolve(value2).isoformat()
+                if value > value2:
+                    raise ValueError('The filter start date must be on or before its end date.')
+        result.append({
+            'column_name': str(column), 'operator_cd': operator,
+            'value_1_txt': value, 'value_2_txt': value2,
+            'kind': 'date' if is_date else entry.get('kind'), 'time_zone': zone_name,
+        })
+
+    snapshot = config.get('date') or {}
+    bounds = {}
+    for side in ('start', 'end'):
+        selection = int(snapshot.get(side + 'SelectionDR') or 0)
+        if not selection:
+            continue
+        if not snapshot.get('filterObject'):
+            raise ValueError('Choose a column for the date range.')
+        if selection == 1:
+            value = today
+        elif selection == 2:
+            value = today - timedelta(days=1)
+        elif selection == 3:
+            days = int(snapshot[side + 'DaysFromToday'])
+            if days < 0:
+                raise ValueError('Days before today cannot be negative.')
+            value = today - timedelta(days=days)
+        elif selection == 4:
+            value = resolve(snapshot[side + 'CustomDate'])
+        else:
+            raise ValueError('Unsupported date range selection.')
+        bounds[side] = value
+        result.append({
+            'column_name': str(snapshot['filterObject']),
+            'operator_cd': 'gte' if side == 'start' else 'lte',
+            'value_1_txt': value.isoformat(), 'kind': 'date', 'time_zone': zone_name,
+        })
+    if 'start' in bounds and 'end' in bounds and bounds['start'] > bounds['end']:
+        raise ValueError('The filter start date must be on or before its end date.')
+    return result
+
+
+def apply_data_filters(results, data_filters):
+    if not data_filters:
+        return results
+
+    # Connectors may return either:
+    # 1) a bare list of rows, or
+    # 2) an envelope: {"data": [...], "next_page": ...}
+    is_envelope = isinstance(results, dict) and isinstance(results.get("data"), list)
+    rows = results.get("data") if is_envelope else results
+    if not isinstance(rows, list):
+        return results
+
+    # 🔹 UI → backend operator mapping
+    OPERATOR_MAP = {
+        # text
+        "text_contains": "CONTAINS",
+        "text_not_contains": "NOT_CONTAINS",
+        "text_starts_with": "STARTS_WITH",
+        "text_ends_with": "ENDS_WITH",
+
+        # equality
+        "is_equal": "EQ",
+        "is_not_equal": "NEQ",
+        "date_is": "EQ",
+        "date_before": "LT",
+        "date_after": "GT",
+
+        # empty
+        "is_empty": "IS_EMPTY",
+        "is_not_empty": "IS_NOT_EMPTY",
+
+        # numeric
+        "gt": "GT",
+        "gte": "GTE",
+        "lt": "LT",
+        "lte": "LTE",
+
+        # range
+        "between": "BETWEEN",
+        "not_between": "NOT_BETWEEN",
+
+        # ignore
+        "none": None,
+    }
+
+    def try_parse_number(val):
+        try:
+            return float(val)
+        except:
+            return val
+
+    def normalize(row_val, v1, v2=None):
+        row_num = try_parse_number(row_val)
+        v1_num = try_parse_number(v1)
+        v2_num = try_parse_number(v2) if v2 is not None else None
+
+        # If both are numbers → compare as numbers
+        if isinstance(row_num, float) and isinstance(v1_num, float):
+            return row_num, v1_num, v2_num
+
+        # Otherwise → compare as strings
+        return str(row_val), str(v1), str(v2) if v2 is not None else None
+
+    def match(row, f):
+        ui_operator = f.get("operator_cd")
+        operator = OPERATOR_MAP.get(ui_operator)
+
+        if operator is None:
+            if ui_operator == 'none':
+                return True
+            raise ValueError('Unsupported sync filter operator.')
+
+        col = f.get("column_name")
+        val1 = f.get("value_1_txt")
+        val2 = f.get("value_2_txt")
+
+        # Some connectors can emit non-dict rows; keep filtering resilient.
+        if not isinstance(row, dict):
+            return False
+        row_val = row.get(col)
+
+        # ---- EMPTY HANDLING ----
+        if operator == "IS_EMPTY":
+            return row_val is None or str(row_val).strip() == ""
+
+        if operator == "IS_NOT_EMPTY":
+            return row_val is not None and str(row_val).strip() != ""
+
+        if row_val is None:
+            return False
+
+        # Normalize values
+        if f.get('kind') == 'date' or ui_operator.startswith('date_'):
+            zone = ZoneInfo(f.get('time_zone') or 'UTC')
+            v1 = _calendar_date(val1, zone)
+            v2 = _calendar_date(val2, zone) if val2 not in (None, '') else None
+            try:
+                row_val = _calendar_date(row_val, zone)
+            except (ValueError, TypeError):
+                return False
+        else:
+            row_val, v1, v2 = normalize(row_val, val1, val2)
+
+        # ---- OPERATORS ----
+
+        if operator == "EQ":
+            return row_val == v1
+
+        elif operator == "NEQ":
+            return row_val != v1
+
+        elif operator == "GT":
+            return row_val > v1
+
+        elif operator == "GTE":
+            return row_val >= v1
+
+        elif operator == "LT":
+            return row_val < v1
+
+        elif operator == "LTE":
+            return row_val <= v1
+
+        elif operator == "BETWEEN":
+            return v1 <= row_val <= v2
+
+        elif operator == "NOT_BETWEEN":
+            return not (v1 <= row_val <= v2)
+
+        elif operator == "CONTAINS":
+            return str(v1).lower() in str(row_val).lower()
+
+        elif operator == "NOT_CONTAINS":
+            return str(v1).lower() not in str(row_val).lower()
+
+        elif operator == "STARTS_WITH":
+            return str(row_val).lower().startswith(str(v1).lower())
+
+        elif operator == "ENDS_WITH":
+            return str(row_val).lower().endswith(str(v1).lower())
+
+        return True
+
+    # 🔹 Apply filters (AND logic)
+    filtered_rows = [row for row in rows if all(match(row, f) for f in data_filters)]
+    if is_envelope:
+        return {**results, "data": filtered_rows}
+    return filtered_rows
+
+

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_sdk/src/mapping.py

@@ -1,5 +1,6 @@
 from dc_sdk import errors
 import logging
+import inspect
 from dc_sdk.src.services.loader import load_connector
 from dc_sdk.src.connection_status import (
     STATUS_CONNECTED,
@@ -14,6 +15,7 @@ class Mapping():
     def __init__(self, credentials):
         Connector = load_connector()
         self.connector = Connector(credentials)
+        self.request_phase = "authentication"
 
     def _set_connection_status(self, status):
         if self.connector.credentials is None:
@@ -35,6 +37,7 @@ class Mapping():
             raise errors.AuthenticationError(message="Failed to initialize connection session")
 
     def _ensure_session(self, force_authenticate=False):
+        self.request_phase = "authentication"
         credentials = self.connector.credentials or {}
         status = credentials.get("last_status_dsc")
 
@@ -71,16 +74,31 @@ class Mapping():
 
         self._ensure_session(force_authenticate=False)
 
-    def _get_metadata(self, required=False):
+    def _discover(self, method_name, sections=None):
+        """Pass sections only to connectors that explicitly support the keyword."""
+        self.request_phase = "metadata"
+        method = getattr(self.connector, method_name)
+        if sections is None:
+            return method()
+        if not isinstance(sections, list) or any(not isinstance(section, str) or not section.strip() for section in sections):
+            raise ValueError("sections must be a list of non-empty strings")
+        parameter = inspect.signature(method).parameters.get("sections")
+        if parameter and parameter.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY):
+            return method(sections=list(dict.fromkeys(sections)))
+        # Legacy connectors retain full discovery and their original response shape.
+        return method()
+
+    def _get_metadata(self, required=False, sections=None):
         """
         Fetch connector metadata.
 
-        On connect/authenticate paths, failures must surface (required=True) so
-        report-catalog connectors cannot look Connected with empty accounts.
+        Required discovery failures still surface to the caller, but do not
+        invalidate successful authentication unless credentials were rejected.
         Soft-fail (required=False) only for optional metadata enrichment.
         """
+        self.request_phase = "metadata"
         try:
-            return self.connector.get_metadata()
+            return self._discover("get_metadata", sections)
         except errors.NotImplementedError:
             return None
         except Exception:
@@ -89,13 +107,13 @@ class Mapping():
             logger.exception("Error getting metadata")
             return None
 
-    def authenticate(self, force_authenticate=True):
+    def authenticate(self, force_authenticate=True, include_metadata=True):
         results = None
         message = None
 
         self._ensure_session(force_authenticate=force_authenticate)
 
-        metadata = self._get_metadata(required=True)
+        metadata = self._get_metadata(required=True) if include_metadata else None
         results = {
             "metadata": metadata
         }
@@ -103,7 +121,7 @@ class Mapping():
 
         return [results, message]
 
-    def connect_get_objects(self, skip_authenticate=False, force_authenticate=False):
+    def connect_get_objects(self, skip_authenticate=False, force_authenticate=False, include_metadata=True):
         results = None
         metadata = None
         objects = None
@@ -114,8 +132,9 @@ class Mapping():
             force_authenticate=force_authenticate,
         )
 
+        self.request_phase = "metadata"
         objects = self.connector.get_objects()
-        metadata = self._get_metadata(required=True)
+        metadata = self._get_metadata(required=True) if include_metadata else None
 
         results = {
             "metadata": metadata,
@@ -125,22 +144,23 @@ class Mapping():
 
         return [results, message]
 
-    def get_metadata_only(self, skip_authenticate=False, force_authenticate=False):
+    def get_metadata_only(self, skip_authenticate=False, force_authenticate=False, sections=None):
         self._resolve_session(
             skip_authenticate=skip_authenticate,
             force_authenticate=force_authenticate,
         )
 
-        metadata = self._get_metadata(required=True)
+        metadata = self._get_metadata(required=True, sections=sections)
         return [{"metadata": metadata}, "Retrieved metadata"]
 
-    def get_objects_only(self, skip_authenticate=False, force_authenticate=False, include_metadata=True):
+    def get_objects_only(self, skip_authenticate=False, force_authenticate=False, include_metadata=True, sections=None):
         self._resolve_session(
             skip_authenticate=skip_authenticate,
             force_authenticate=force_authenticate,
         )
 
-        objects = self.connector.get_objects()
+        self.request_phase = "metadata"
+        objects = self._discover("get_objects", sections)
         results = {"objects": objects}
 
         if include_metadata:

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_sdk/src/pipeline.py

@@ -10,6 +10,7 @@ from .services.loader import load_connector
 from .destination_object_template import resolve_destination_object_template
 from dc_sdk.data_stream import DataStream
 from dc_sdk.json_safe import to_json_safe
+from dc_sdk.row_filters import compile_sync_filters, apply_data_filters
 
 TEMP_UPLOADS = 'temporary-files'
 SUB_FOLDER = 'etlJobHistory'
@@ -82,6 +83,8 @@ class PipelineConductor:
         while self.authentication_tries <= 3 and not authenticated:
             try:
                 authenticated = self.connector.authenticate()
+                if not authenticated:
+                    raise errors.AuthenticationError("Connector authentication failed.")
             except Exception as e:
                 if self.authentication_tries == 3 or "firewall" in str(e): # todo: fix add retry flg in raised errors
                     raise e
@@ -97,9 +100,9 @@ class PipelineConductor:
         response = self.api.get_credential_updates(True, self.pipeline_id)
 
         creds = self.aws.decrypt_customer_data_object(response['credential'], response['organization'], 
-            encrypted_data_key_txt=self.pipeline_details.source_credential_information.get('encrypted_data_key_txt'),
-            encryption_iv_txt=self.pipeline_details.source_credential_information.get('encryption_iv_txt'),
-            encryption_auth_tag_txt=self.pipeline_details.source_credential_information.get('encryption_auth_tag_txt')
+            encrypted_data_key_txt=response.get('encrypted_data_key_txt'),
+            encryption_iv_txt=response.get('encryption_iv_txt'),
+            encryption_auth_tag_txt=response.get('encryption_auth_tag_txt')
         )
         
         self.connector.credentials = creds
@@ -108,9 +111,9 @@ class PipelineConductor:
         response = self.api.get_credential_updates(False, self.pipeline_id)
 
         creds = self.aws.decrypt_customer_data_object(response['credential'], response['organization'], 
-            encrypted_data_key_txt=self.pipeline_details.destination_credential_information.get('encrypted_data_key_txt'),
-            encryption_iv_txt=self.pipeline_details.destination_credential_information.get('encryption_iv_txt'),
-            encryption_auth_tag_txt=self.pipeline_details.destination_credential_information.get('encryption_auth_tag_txt')
+            encrypted_data_key_txt=response.get('encrypted_data_key_txt'),
+            encryption_iv_txt=response.get('encryption_iv_txt'),
+            encryption_auth_tag_txt=response.get('encryption_auth_tag_txt')
         )
         
         self.connector.credentials = creds    
@@ -124,6 +127,8 @@ class PipelineConductor:
         while self.authentication_tries <= 3 and not authenticated:
             try:
                 authenticated = self.connector.authenticate()
+                if not authenticated:
+                    raise errors.AuthenticationError("Connector authentication failed.")
             except Exception as e:
                 if self.authentication_tries == 3 or "firewall" in str(e): # todo: fix add retry flg in raised errors
                     raise e
@@ -137,6 +142,7 @@ class PipelineConductor:
 
     def get_data(self):
         # Determine batch size
+        self._get_data_filters()  # Validate and resolve relative dates before fetching any rows.
 
         self.log(self.log_templates.GET_DATA_START.format(
             self.pipeline_details.source_object_id,
@@ -146,31 +152,22 @@ class PipelineConductor:
         nrows = self._get_batch_row_count()
         max_allowed = self.pipeline_details.max_allowed_retrieval
 
-        results = self.connector.get_data(self.pipeline_details.source_object_id, self._get_field_ids(), n_rows=nrows, filters=self._get_filters(), options=self.pipeline_details.options)
-
-        while "next_page" in results and results["next_page"] != None:
-            if "data" in results and results["data"] != None and results["data"] != []:
-                self._update_extracted_sync_cursor(results.get("metadata"))
-                limit_reached = self._process_rows(results["data"], max_allowed)
-                if limit_reached:
-                    break
-            elif results["data"] != []:
-                self.internal_log(self.log_templates.INTERNAL_GET_DATA_FETCHED.format(0))
-            
-            # Check if we've reached the limit before fetching next page
+        next_page = None
+        while True:
+            results = self.connector.get_data(
+                self.pipeline_details.source_object_id, self._get_source_field_ids(),
+                n_rows=nrows, filters=self._get_filters(),
+                options=self.pipeline_details.options, next_page=next_page,
+            )
+            self._update_extracted_sync_cursor(results.get("metadata"))
+            rows = results.get("data") or []
+            if rows and self._process_rows(rows, max_allowed):
+                break
             if max_allowed is not None and self.row_count >= max_allowed:
                 break
-                
-            if "next_page" in results and results["next_page"] != None:
-                results = self.connector.get_data(self.pipeline_details.source_object_id, self._get_field_ids(), n_rows=nrows, filters=self._get_filters(), options=self.pipeline_details.options, next_page=results["next_page"])
-        
-        if "data" in results and results["data"] != None and results["data"] != []:
-            self._update_extracted_sync_cursor(results.get("metadata"))
-            self._process_rows(results["data"], max_allowed)
-        elif results["data"] != []:
-            self.internal_log(self.log_templates.INTERNAL_GET_DATA_FETCHED.format(0))
-        else:
-            self._update_extracted_sync_cursor(results.get("metadata"))
+            next_page = results.get("next_page")
+            if next_page is None:
+                break
 
         if self._should_track_sync_cursor():
             self._save_pending_sync_cursor()
@@ -184,6 +181,8 @@ class PipelineConductor:
         Not wired into the default SOURCE path yet — call explicitly when testing.
         Retains extract/normalize hints via a sibling .meta.json object.
         """
+        if self._get_data_filters():
+            raise errors.DataError('Row filters require the standard row transfer path.')
         if not hasattr(self.connector, "get_data_stream"):
             raise errors.DataError(
                 "Connector does not implement get_data_stream; cannot use stream transfer."
@@ -233,6 +232,7 @@ class PipelineConductor:
             if not loaded:
                 raise errors.LoadDataError("Loading data failed.")
         else:
+            keys.sort(key=self._transfer_batch_sort_key)
             for index, key in enumerate(keys):
                 file_object = None
                 if PipelineEnvironment.platform == "aws":
@@ -257,6 +257,11 @@ class PipelineConductor:
         self.log(self.log_templates.LOAD_DATA_FINISHED.format(self.row_count, destination_object_id))
         self._persist_sync_cursor_after_load()
 
+    @staticmethod
+    def _transfer_batch_sort_key(key):
+        match = re.search(r"-b(\d+)\.json$", key)
+        return (0, int(match.group(1))) if match else (1, key)
+
     def _create_connector(self, connector_cls, credentials):
         init_params = inspect.signature(connector_cls.__init__).parameters
         if "pipeline_context" in init_params:
@@ -428,6 +433,14 @@ class PipelineConductor:
         self.api.create_pipeline_mapping(self.pipeline_id, self.pipeline_details.pipeline_mapping_json)
 
     def _process_rows(self, rows, max_allowed=None):
+        rows = apply_data_filters(rows, self._get_data_filters())
+        if not rows:
+            return False
+        selected_ids = {str(field) for field in self._get_field_ids()}
+        if selected_ids:
+            filter_only = {f['column_name'] for f in self._get_data_filters()} - selected_ids
+            if filter_only:
+                rows = [{key: value for key, value in row.items() if str(key) not in filter_only} for row in rows]
         # Check if we have a limit and need to truncate rows
         limit_reached = False
         if max_allowed is not None:
@@ -590,7 +603,7 @@ class PipelineConductor:
     def _get_batch_row_count(self):
         results = self.connector.get_data(
             self.pipeline_details.source_object_id,
-            self._get_field_ids(),
+            self._get_source_field_ids(),
             n_rows=32,
             filters=self._get_filters(),
             options=self.pipeline_details.options
@@ -639,6 +652,9 @@ class PipelineConductor:
         return batch_size
 
     def _get_filters(self):
+        # New filters are enforced centrally; do not also apply a stale legacy range.
+        if 'sync_filters' in (self.pipeline_details.options or {}):
+            return None
         if self.mode == "prod":
             return {
                 "filtered_column_nm": self.pipeline_details.filtered_column_nm,
@@ -651,6 +667,22 @@ class PipelineConductor:
         else: 
             return self.filters
 
+    def _get_data_filters(self):
+        if not hasattr(self, '_resolved_data_filters'):
+            self._resolved_data_filters = compile_sync_filters(
+                (self.pipeline_details.options or {}).get('sync_filters')
+            )
+        return self._resolved_data_filters
+
+    def _get_source_field_ids(self):
+        fields = self._get_field_ids()
+        if not fields:  # An empty mapping requests every source column.
+            return fields
+        ids = {str(field) for field in fields}
+        return fields + list(dict.fromkeys(
+            f['column_name'] for f in self._get_data_filters() if f['column_name'] not in ids
+        ))
+
     def _get_field_ids(self):
         if not self.pipeline_details.pipeline_mapping_json:
             return []
@@ -745,4 +777,4 @@ class PipelineConductor:
                 i += 1
             seen[name] = True
             mapping.append({"column": name, "mapped": fid})
-        return mapping
+        return mapping

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_sdk/src/services/api.py

@@ -103,6 +103,21 @@ class DataConnectorAPI:
             f"credentials/{pipeline_id}?source={'true' if source_flg else 'false'}"
         )
 
+        envelope_fields = (
+            'encrypted_data_key_txt', 'encryption_iv_txt', 'encryption_auth_tag_txt'
+        )
+        if not all(field in response for field in envelope_fields):
+            # Older APIs omit the envelope. Fetch ciphertext and metadata together
+            # from a fresh pipeline snapshot; never mix separate credential reads.
+            details = self.get(str(pipeline_id))
+            side = 'source' if source_flg else 'destination'
+            information = details.get(f'{side}_credential_information') or {}
+            response = {
+                'credential': details[f'{side}_encryption_credential_txt'],
+                'organization': details['customer_metadata_uuid'],
+                **{field: information.get(field) for field in envelope_fields},
+            }
+
         return response
 
     def create_pipeline_mapping(self, pipeline_id, pipeline_mapping_json):
@@ -220,4 +235,4 @@ class DataConnectorAPI:
     def _get_retry_delay(self, attempt):
         base_delay = min(2 ** attempt, 10)
         jitter = random.uniform(0, 0.5)
-        return base_delay + jitter
+        return base_delay + jitter

{dc_python_sdk-1.5.54 → dc_python_sdk-1.6.1}/src/dc_sdk/src/services/aws.py

@@ -52,19 +52,16 @@ class AwsService:
         self._ecs_cluster_name = APP_ENV if APP_ENV != "local" else "development"
 
     def get_keys(self, pipeline_run_history_id):
-        keys = []
-        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"transfers/e{pipeline_run_history_id}")
-        if 'Contents' in response:
-            for key in response['Contents']:
-                keys.append(key['Key'])
-        return keys
+        return self._get_keys(f"transfers/e{pipeline_run_history_id}-b")
 
     def get_dev_keys(self, prefix):
+        return self._get_keys(f"devTransfers/e{prefix}-b")
+
+    def _get_keys(self, prefix):
         keys = []
-        response = s3_client.list_objects(Bucket=self.s3_bucket, Prefix=f"devTransfers/e{prefix}")
-        if 'Contents' in response:
-            for key in response['Contents']:
-                keys.append(key['Key'])
+        paginator = s3_client.get_paginator('list_objects_v2')
+        for page in paginator.paginate(Bucket=self.s3_bucket, Prefix=prefix):
+            keys.extend(obj['Key'] for obj in page.get('Contents', []))
         return keys
 
     def download_object(self, key_name):
@@ -328,4 +325,4 @@ class EncryptionService:
             encryption_auth_tag_txt=encryption_auth_tag_txt
         )
 
-        return json.loads(data)
+        return json.loads(data)

dc_python_sdk-1.5.54/src/dc_sdk/handler.py

@@ -1,242 +0,0 @@
-from dc_sdk.errors import Error
-from dc_sdk.json_safe import to_json_safe
-from dc_sdk.src.mapping import Mapping
-from dc_sdk.src.connection_status import STATUS_ERROR
-import traceback
-from importlib.metadata import version
-
-def get_action_name(action: int) -> str:
-    return {
-        0: "authenticating",
-        1: "retrieving fields",
-        2: "retrieving 5 row preview",
-        3: "testing connection",
-        4: "starting the connector",
-        5: "retrieving metadata",
-        6: "retrieving objects",
-    }[action]
-
-def apply_data_filters(results, data_filters):
-    if not data_filters:
-        return results
-
-    # Connectors may return either:
-    # 1) a bare list of rows, or
-    # 2) an envelope: {"data": [...], "next_page": ...}
-    is_envelope = isinstance(results, dict) and isinstance(results.get("data"), list)
-    rows = results.get("data") if is_envelope else results
-    if not isinstance(rows, list):
-        return results
-
-    # 🔹 UI → backend operator mapping
-    OPERATOR_MAP = {
-        # text
-        "text_contains": "CONTAINS",
-        "text_not_contains": "NOT_CONTAINS",
-        "text_starts_with": "STARTS_WITH",
-        "text_ends_with": "ENDS_WITH",
-
-        # equality
-        "is_equal": "EQ",
-        "is_not_equal": "NEQ",
-
-        # empty
-        "is_empty": "IS_EMPTY",
-        "is_not_empty": "IS_NOT_EMPTY",
-
-        # numeric
-        "gt": "GT",
-        "gte": "GTE",
-        "lt": "LT",
-        "lte": "LTE",
-
-        # range
-        "between": "BETWEEN",
-        "not_between": "NOT_BETWEEN",
-
-        # ignore
-        "none": None,
-    }
-
-    def try_parse_number(val):
-        try:
-            return float(val)
-        except:
-            return val
-
-    def normalize(row_val, v1, v2=None):
-        row_num = try_parse_number(row_val)
-        v1_num = try_parse_number(v1)
-        v2_num = try_parse_number(v2) if v2 is not None else None
-
-        # If both are numbers → compare as numbers
-        if isinstance(row_num, float) and isinstance(v1_num, float):
-            return row_num, v1_num, v2_num
-
-        # Otherwise → compare as strings
-        return str(row_val), str(v1), str(v2) if v2 is not None else None
-
-    def match(row, f):
-        ui_operator = f.get("operator_cd")
-        operator = OPERATOR_MAP.get(ui_operator)
-
-        if operator is None:
-            return True  # skip "none"
-
-        col = f.get("column_name")
-        val1 = f.get("value_1_txt")
-        val2 = f.get("value_2_txt")
-
-        # Some connectors can emit non-dict rows; keep filtering resilient.
-        if not isinstance(row, dict):
-            return False
-        row_val = row.get(col)
-
-        # ---- EMPTY HANDLING ----
-        if operator == "IS_EMPTY":
-            return row_val is None or str(row_val).strip() == ""
-
-        if operator == "IS_NOT_EMPTY":
-            return row_val is not None and str(row_val).strip() != ""
-
-        if row_val is None:
-            return False
-
-        # Normalize values
-        row_val, v1, v2 = normalize(row_val, val1, val2)
-
-        # ---- OPERATORS ----
-
-        if operator == "EQ":
-            return row_val == v1
-
-        elif operator == "NEQ":
-            return row_val != v1
-
-        elif operator == "GT":
-            return row_val > v1
-
-        elif operator == "GTE":
-            return row_val >= v1
-
-        elif operator == "LT":
-            return row_val < v1
-
-        elif operator == "LTE":
-            return row_val <= v1
-
-        elif operator == "BETWEEN":
-            return v1 <= row_val <= v2
-
-        elif operator == "NOT_BETWEEN":
-            return not (v1 <= row_val <= v2)
-
-        elif operator == "CONTAINS":
-            return str(v1).lower() in str(row_val).lower()
-
-        elif operator == "NOT_CONTAINS":
-            return str(v1).lower() not in str(row_val).lower()
-
-        elif operator == "STARTS_WITH":
-            return str(row_val).lower().startswith(str(v1).lower())
-
-        elif operator == "ENDS_WITH":
-            return str(row_val).lower().endswith(str(v1).lower())
-
-        return True
-
-    # 🔹 Apply filters (AND logic)
-    filtered_rows = [row for row in rows if all(match(row, f) for f in data_filters)]
-    if is_envelope:
-        return {**results, "data": filtered_rows}
-    return filtered_rows
-
-
-def handler(event, context):
-    print("version: ", version("dc-python-sdk"))
-    """Lambda Handler"""
-    action_name = None
-    internal_error = False
-    message = None
-    results = None
-    error = None
-    mapping = None
-    action = int(event['action']) if 'action' in event else 4
-    get_objects = event.get('get_objects', True)
-    skip_authenticate = event.get('skip_authenticate', False)
-    force_authenticate = event.get('force_authenticate', False)
-    include_metadata = event.get('include_metadata', True)
-    credentials_dict = event['credentials'] if 'credentials' in event else None
-    object_id = event['object_id'] if 'object_id' in event else None
-    field_ids = event['mapping'] if 'mapping' in event else None
-    options = event['options'] if 'options' in event else dict()
-    next_page = event['next_page'] if 'next_page' in event else None
-    n_rows = event['n_rows'] if 'n_rows' in event else None
-    filters = event['filters'] if 'filters' in event else None
-    data_filters = event['data_filters'] if 'data_filters' in event else None
-
-    try:
-        action_name = get_action_name(action)
-        mapping = Mapping(credentials_dict)
-
-        if action == 0:
-            if get_objects:
-                results, message = mapping.connect_get_objects(
-                    skip_authenticate=skip_authenticate,
-                    force_authenticate=force_authenticate,
-                )
-            else:
-                results, message = mapping.authenticate(
-                    force_authenticate=force_authenticate or not skip_authenticate,
-                )
-        elif action == 1:
-            results, message = mapping.get_fields(object_id, options)
-        elif action == 2:
-            results, message = mapping.get_five_row_preview(
-                object_id, field_ids, options, n_rows, filters, next_page)
-            results = apply_data_filters(results, data_filters)
-        elif action == 3:
-            results, message = mapping.test_connection()
-        elif action == 5:
-            results, message = mapping.get_metadata_only(
-                skip_authenticate=skip_authenticate,
-                force_authenticate=force_authenticate,
-            )
-        elif action == 6:
-            results, message = mapping.get_objects_only(
-                skip_authenticate=skip_authenticate,
-                force_authenticate=force_authenticate,
-                include_metadata=include_metadata,
-            )
-        else:
-            raise Error("Invalid action", "InvalidActionError")
-    except Error as e:
-        message = e.message or str(e)
-        internal_error = e.internal
-        error_trace = traceback.format_exc()
-        error = error_trace
-        if mapping:
-            mapping._set_connection_status(STATUS_ERROR)
-
-    except Exception as e:
-        error_trace = traceback.format_exc()
-        error = error_trace
-        internal_error = True
-        if mapping:
-            mapping._set_connection_status(STATUS_ERROR)
-
-    if error:
-        print(error)
-
-    last_status_dsc = None
-    if mapping and mapping.connector.credentials:
-        last_status_dsc = mapping.connector.credentials.get("last_status_dsc")
-
-    return {
-        'message': message if not internal_error else f"Something went wrong with {action_name}",
-        'results': to_json_safe(results),
-        'credentials': mapping.connector.credentials if mapping else None,
-        'last_status_dsc': last_status_dsc,
-        'error': error,
-        'internal_error': internal_error
-    }