ingestr 0.8.4__py3-none-any.whl → 0.9.1__py3-none-any.whl

ingestr/src/zendesk/__init__.py

@@ -0,0 +1,460 @@
+"""
+Defines all the sources and resources needed for ZendeskSupport, ZendeskChat and ZendeskTalk
+"""
+
+from itertools import chain
+from typing import Iterable, Iterator, Optional
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem, TDataItems
+from dlt.sources import DltResource
+
+from .helpers.api_helpers import process_ticket, process_ticket_field
+from .helpers.credentials import TZendeskCredentials, ZendeskCredentialsOAuth
+from .helpers.talk_api import PaginationType, ZendeskAPIClient
+from .settings import (
+    CUSTOM_FIELDS_STATE_KEY,
+    DEFAULT_START_DATE,
+    INCREMENTAL_TALK_ENDPOINTS,
+    SUPPORT_ENDPOINTS,
+    SUPPORT_EXTRA_ENDPOINTS,
+    TALK_ENDPOINTS,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_talk(
+    credentials: TZendeskCredentials = dlt.secrets.value,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from Zendesk Talk for phone calls and voicemails.
+
+    `start_date` argument can be used on its own or together with `end_date`. When both are provided
+    data is limited to items updated in that time range.
+    The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+    All resources opt-in to use Airflow scheduler if run as Airflow task
+
+    Args:
+        credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+        end_date: The end time of the range for which to load data.
+            If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+    Yields:
+        DltResource: Data resources from Zendesk Talk.
+    """
+
+    # use the credentials to authenticate with the ZendeskClient
+    zendesk_client = ZendeskAPIClient(credentials)
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    # regular endpoints
+    for key, talk_endpoint, item_name, cursor_paginated in TALK_ENDPOINTS:
+        yield dlt.resource(
+            talk_resource(
+                zendesk_client,
+                key,
+                item_name or talk_endpoint,
+                PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+            ),
+            name=key,
+            write_disposition="replace",
+        )
+
+    # adding incremental endpoints
+    for key, talk_incremental_endpoint in INCREMENTAL_TALK_ENDPOINTS.items():
+        yield dlt.resource(
+            talk_incremental_resource,
+            name=f"{key}_incremental",
+            primary_key="id",
+            write_disposition="merge",
+        )(
+            zendesk_client=zendesk_client,
+            talk_endpoint_name=key,
+            talk_endpoint=talk_incremental_endpoint,
+            updated_at=dlt.sources.incremental[str](
+                "updated_at",
+                initial_value=start_date_obj.isoformat(),
+                end_value=end_date_obj.isoformat() if end_date_obj else None,
+                allow_external_schedulers=True,
+            ),
+        )
+
+
+def talk_resource(
+    zendesk_client: ZendeskAPIClient,
+    talk_endpoint_name: str,
+    talk_endpoint: str,
+    pagination_type: PaginationType,
+) -> Iterator[TDataItem]:
+    """
+    Loads data from a Zendesk Talk endpoint.
+
+    Args:
+        zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+        talk_endpoint_name: The name of the talk_endpoint.
+        talk_endpoint: The actual URL ending of the endpoint.
+        pagination: Type of pagination type used by endpoint
+
+    Yields:
+        TDataItem: Dictionary containing the data from the endpoint.
+    """
+    # send query and process it
+    yield from zendesk_client.get_pages(
+        talk_endpoint, talk_endpoint_name, pagination_type
+    )
+
+
+def talk_incremental_resource(
+    zendesk_client: ZendeskAPIClient,
+    talk_endpoint_name: str,
+    talk_endpoint: str,
+    updated_at: dlt.sources.incremental[str],
+) -> Iterator[TDataItem]:
+    """
+    Loads data from a Zendesk Talk endpoint with incremental loading.
+
+    Args:
+        zendesk_client: An instance of ZendeskAPIClient for making API calls to Zendesk Talk.
+        talk_endpoint_name: The name of the talk_endpoint.
+        talk_endpoint: The actual URL ending of the endpoint.
+        updated_at: Source for the last updated timestamp.
+
+    Yields:
+        TDataItem: Dictionary containing the data from the endpoint.
+    """
+    # send the request and process it
+    for page in zendesk_client.get_pages(
+        talk_endpoint,
+        talk_endpoint_name,
+        PaginationType.START_TIME,
+        params={
+            "start_time": ensure_pendulum_datetime(updated_at.last_value).int_timestamp
+        },
+    ):
+        yield page
+        if updated_at.end_out_of_range:
+            return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_chat(
+    credentials: ZendeskCredentialsOAuth,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from Zendesk Chat for chat interactions.
+
+    `start_date` argument can be used on its own or together with `end_date`. When both are provided
+    data is limited to items updated in that time range.
+    The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+    All resources opt-in to use Airflow scheduler if run as Airflow task
+
+    Args:
+        credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+        end_date: The end time of the range for which to load data.
+            If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+    Yields:
+        DltResource: Data resources from Zendesk Chat.
+    """
+
+    # Authenticate
+    zendesk_client = ZendeskAPIClient(credentials, url_prefix="https://www.zopim.com")
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    yield dlt.resource(chats_table_resource, name="chats", write_disposition="merge")(
+        zendesk_client,
+        dlt.sources.incremental[str](
+            "update_timestamp|updated_timestamp",
+            initial_value=start_date_obj.isoformat(),
+            end_value=end_date_obj.isoformat() if end_date_obj else None,
+            allow_external_schedulers=True,
+        ),
+    )
+
+
+def chats_table_resource(
+    zendesk_client: ZendeskAPIClient,
+    update_timestamp: dlt.sources.incremental[str],
+) -> Iterator[TDataItems]:
+    """
+    Resource for Chats
+
+    Args:
+        zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+        update_timestamp: Incremental source specifying the timestamp for incremental loading.
+
+    Yields:
+        dict: A dictionary representing each row of data.
+    """
+    chat_pages = zendesk_client.get_pages(
+        "/api/v2/incremental/chats",
+        "chats",
+        PaginationType.START_TIME,
+        params={
+            "start_time": ensure_pendulum_datetime(
+                update_timestamp.last_value
+            ).int_timestamp,
+            "fields": "chats(*)",
+        },
+    )
+    for page in chat_pages:
+        yield page
+
+        if update_timestamp.end_out_of_range:
+            return
+
+
+@dlt.source(max_table_nesting=0)
+def zendesk_support(
+    credentials: TZendeskCredentials,
+    load_all: bool = True,
+    pivot_ticket_fields: bool = True,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from Zendesk Support for tickets, users, brands, organizations, and groups.
+
+    `start_date` argument can be used on its own or together with `end_date`. When both are provided
+    data is limited to items updated in that time range.
+    The range is "half-open", meaning elements equal and higher than `start_date` and elements lower than `end_date` are included.
+    All resources opt-in to use Airflow scheduler if run as Airflow task
+
+    Args:
+        credentials: The credentials for authentication. Defaults to the value in the `dlt.secrets` object.
+        load_all: Whether to load extra resources for the API. Defaults to True.
+        pivot_ticket_fields: Whether to pivot the custom fields in tickets. Defaults to True.
+        start_date: The start time of the range for which to load. Defaults to January 1st 2000.
+        end_date: The end time of the range for which to load data.
+            If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+
+    Returns:
+        Sequence[DltResource]: Multiple dlt resources.
+    """
+
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    start_date_ts = start_date_obj.int_timestamp
+    start_date_iso_str = start_date_obj.isoformat()
+    end_date_ts: Optional[int] = None
+    end_date_iso_str: Optional[str] = None
+    if end_date_obj:
+        end_date_ts = end_date_obj.int_timestamp
+        end_date_iso_str = end_date_obj.isoformat()
+
+    @dlt.resource(primary_key="id", write_disposition="append")
+    def ticket_events(
+        zendesk_client: ZendeskAPIClient,
+        timestamp: dlt.sources.incremental[int] = dlt.sources.incremental(
+            "timestamp",
+            initial_value=start_date_ts,
+            end_value=end_date_ts,
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[TDataItem]:
+        # URL For ticket events
+        # 'https://d3v-dlthub.zendesk.com/api/v2/incremental/ticket_events.json?start_time=946684800'
+        event_pages = zendesk_client.get_pages(
+            "/api/v2/incremental/ticket_events.json",
+            "ticket_events",
+            PaginationType.STREAM,
+            params={"start_time": timestamp.last_value},
+        )
+        for page in event_pages:
+            yield page
+            if timestamp.end_out_of_range:
+                return
+
+    @dlt.resource(
+        name="tickets",
+        primary_key="id",
+        write_disposition="merge",
+        columns={
+            "tags": {"data_type": "complex"},
+            "custom_fields": {"data_type": "complex"},
+        },
+    )
+    def ticket_table(
+        zendesk_client: ZendeskAPIClient,
+        pivot_fields: bool = True,
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[TDataItem]:
+        """
+        Resource for tickets table. Uses DLT state to handle column renaming of custom fields to prevent changing the names of said columns.
+        This resource uses pagination, loading and side loading to make API calls more efficient.
+
+        Args:
+            zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+            pivot_fields: Indicates whether to pivot the custom fields in tickets. Defaults to True.
+            per_page: The number of Ticket objects to load per page. Defaults to 1000.
+            updated_at: Incremental source for the 'updated_at' column.
+                Defaults to dlt.sources.incremental("updated_at", initial_value=start_date).
+
+        Yields:
+            TDataItem: Dictionary containing the ticket data.
+        """
+        # grab the custom fields from dlt state if any
+        if pivot_fields:
+            load_ticket_fields_state(zendesk_client)
+        fields_dict = dlt.current.source_state().setdefault(CUSTOM_FIELDS_STATE_KEY, {})
+        # include_objects = ["users", "groups", "organisation", "brands"]
+        ticket_pages = zendesk_client.get_pages(
+            "/api/v2/incremental/tickets",
+            "tickets",
+            PaginationType.STREAM,
+            params={"start_time": updated_at.last_value.int_timestamp},
+        )
+        for page in ticket_pages:
+            yield [
+                process_ticket(ticket, fields_dict, pivot_custom_fields=pivot_fields)
+                for ticket in page
+            ]
+            # stop loading when using end_value and end is reached
+            if updated_at.end_out_of_range:
+                return
+
+    @dlt.resource(
+        name="ticket_metric_events", primary_key="id", write_disposition="append"
+    )
+    def ticket_metric_table(
+        zendesk_client: ZendeskAPIClient,
+        time: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "time",
+            initial_value=start_date_iso_str,
+            end_value=end_date_iso_str,
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[TDataItem]:
+        """
+        Resource for ticket metric events table. Returns all the ticket metric events from the starting date,
+        with the default starting date being January 1st of the current year.
+
+        Args:
+            zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+            time: Incremental source for the 'time' column,
+                indicating the starting date for retrieving ticket metric events.
+                Defaults to dlt.sources.incremental("time", initial_value=start_date_iso_str).
+
+        Yields:
+            TDataItem: Dictionary containing the ticket metric event data.
+        """
+        # "https://example.zendesk.com/api/v2/incremental/ticket_metric_events?start_time=1332034771"
+        metric_event_pages = zendesk_client.get_pages(
+            "/api/v2/incremental/ticket_metric_events",
+            "ticket_metric_events",
+            PaginationType.CURSOR,
+            params={
+                "start_time": ensure_pendulum_datetime(time.last_value).int_timestamp,
+            },
+        )
+        for page in metric_event_pages:
+            yield page
+
+            if time.end_out_of_range:
+                return
+
+    def ticket_fields_table(zendesk_client: ZendeskAPIClient) -> Iterator[TDataItem]:
+        """
+        Loads ticket fields data from Zendesk API.
+
+        Args:
+            zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+
+        Yields:
+            TDataItem: Dictionary containing the ticket fields data.
+        """
+        # get dlt state
+        ticket_custom_fields = dlt.current.source_state().setdefault(
+            CUSTOM_FIELDS_STATE_KEY, {}
+        )
+        # get all custom fields and update state if needed, otherwise just load dicts into tables
+        all_fields = list(
+            chain.from_iterable(
+                zendesk_client.get_pages(
+                    "/api/v2/ticket_fields.json", "ticket_fields", PaginationType.OFFSET
+                )
+            )
+        )
+        # all_fields = zendesk_client.ticket_fields()
+        for field in all_fields:
+            yield process_ticket_field(field, ticket_custom_fields)
+
+    def load_ticket_fields_state(
+        zendesk_client: ZendeskAPIClient,
+    ) -> None:
+        for _ in ticket_fields_table(zendesk_client):
+            pass
+
+    ticket_fields_resource = dlt.resource(
+        name="ticket_fields", write_disposition="replace"
+    )(ticket_fields_table)
+
+    # Authenticate
+    zendesk_client = ZendeskAPIClient(credentials)
+
+    # loading base tables
+    resource_list = [
+        ticket_fields_resource(zendesk_client=zendesk_client),
+        ticket_events(zendesk_client=zendesk_client),
+        ticket_table(zendesk_client=zendesk_client, pivot_fields=pivot_ticket_fields),
+        ticket_metric_table(zendesk_client=zendesk_client),
+    ]
+
+    # other tables to be loaded
+    resources_to_be_loaded = list(SUPPORT_ENDPOINTS)  # make a copy
+    if load_all:
+        resources_to_be_loaded.extend(SUPPORT_EXTRA_ENDPOINTS)
+    for resource, endpoint_url, data_key, cursor_paginated in resources_to_be_loaded:
+        resource_list.append(
+            dlt.resource(
+                basic_resource(
+                    zendesk_client, endpoint_url, data_key or resource, cursor_paginated
+                ),
+                name=resource,
+                write_disposition="replace",
+            )
+        )
+    return resource_list
+
+
+def basic_resource(
+    zendesk_client: ZendeskAPIClient,
+    endpoint_url: str,
+    data_key: str,
+    cursor_paginated: bool,
+) -> Iterator[TDataItem]:
+    """
+    Basic loader for most endpoints offered by Zenpy. Supports pagination. Expects to be called as a DLT Resource.
+
+    Args:
+        zendesk_client: The Zendesk API client instance, used to make calls to Zendesk API.
+        resource: The Zenpy endpoint to retrieve data from, usually directly linked to a Zendesk API endpoint.
+        cursor_paginated: Tells to use CURSOR pagination or OFFSET/no pagination
+
+    Yields:
+        TDataItem: Dictionary containing the resource data.
+    """
+
+    pages = zendesk_client.get_pages(
+        endpoint_url,
+        data_key,
+        PaginationType.CURSOR if cursor_paginated else PaginationType.OFFSET,
+    )
+    yield from pages

ingestr/src/zendesk/helpers/__init__.py

@@ -0,0 +1,25 @@
+"""Zendesk source helpers"""
+
+from typing import List, Tuple
+
+from dlt.common import pendulum
+from dlt.common.time import timedelta
+
+
+def make_date_ranges(
+    start: pendulum.DateTime, end: pendulum.DateTime, step: timedelta
+) -> List[Tuple[pendulum.DateTime, pendulum.DateTime]]:
+    """Make tuples of (start, end) date ranges between the given `start` and `end` dates.
+    The last range in the resulting list will be capped to the value of `end` argument so it may be smaller than `step`
+
+    Example usage, create 1 week ranges between January 1st 2023 and today:
+    >>> make_date_ranges(pendulum.DateTime(2023, 1, 1).as_tz('UTC'), pendulum.today(), timedelta(weeks=1))
+    """
+    ranges = []
+    while True:
+        end_time = min(start + step, end)
+        ranges.append((start, end_time))
+        if end_time == end:
+            break
+        start = end_time
+    return ranges

ingestr/src/zendesk/helpers/api_helpers.py

@@ -0,0 +1,105 @@
+from typing import Dict, Optional, TypedDict
+
+from dlt.common import logger, pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import DictStrAny, DictStrStr, TDataItem
+
+
+class TCustomFieldInfo(TypedDict):
+    title: str
+    options: DictStrStr
+
+
+def _parse_date_or_none(value: Optional[str]) -> Optional[pendulum.DateTime]:
+    if not value:
+        return None
+    return ensure_pendulum_datetime(value)
+
+
+def process_ticket(
+    ticket: DictStrAny,
+    custom_fields: Dict[str, TCustomFieldInfo],
+    pivot_custom_fields: bool = True,
+) -> DictStrAny:
+    """
+    Helper function that processes a ticket object and returns a dictionary of ticket data.
+
+    Args:
+        ticket: The ticket dict object returned by a Zendesk API call.
+        custom_fields: A dictionary containing all the custom fields available for tickets.
+        pivot_custom_fields: A boolean indicating whether to pivot all custom fields or not.
+            Defaults to True.
+
+    Returns:
+        DictStrAny: A dictionary containing cleaned data about a ticket.
+    """
+    # pivot custom field if indicated as such
+    # get custom fields
+    pivoted_fields = set()
+    for custom_field in ticket["custom_fields"]:
+        if pivot_custom_fields:
+            cus_field_id = str(custom_field["id"])
+            field = custom_fields.get(cus_field_id, None)
+            if field is None:
+                logger.warning(
+                    "Custom field with ID %s does not exist in fields state. It may have been created after the pipeline run started.",
+                    cus_field_id,
+                )
+                custom_field["ticket_id"] = ticket["id"]
+                continue
+
+            pivoted_fields.add(cus_field_id)
+            field_name = field["title"]
+            current_value = custom_field["value"]
+            options = field["options"]
+            # Map dropdown values to labels
+            if not current_value or not options:
+                ticket[field_name] = current_value
+            elif isinstance(
+                current_value, list
+            ):  # Multiple choice field has a list of values
+                ticket[field_name] = [options.get(key, key) for key in current_value]
+            else:
+                ticket[field_name] = options.get(current_value)
+        else:
+            custom_field["ticket_id"] = ticket["id"]
+    # delete fields that are not needed for pivoting
+    if pivot_custom_fields:
+        ticket["custom_fields"] = [
+            f for f in ticket["custom_fields"] if str(f["id"]) not in pivoted_fields
+        ]
+        if not ticket["custom_fields"]:
+            del ticket["custom_fields"]
+    del ticket["fields"]
+    # modify dates to return datetime objects instead
+    ticket["updated_at"] = _parse_date_or_none(ticket["updated_at"])
+    ticket["created_at"] = _parse_date_or_none(ticket["created_at"])
+    ticket["due_at"] = _parse_date_or_none(ticket["due_at"])
+    return ticket
+
+
+def process_ticket_field(
+    field: DictStrAny, custom_fields_state: Dict[str, TCustomFieldInfo]
+) -> TDataItem:
+    """Update custom field mapping in dlt state for the given field."""
+    # grab id and update state dict
+    # if the id is new, add a new key to indicate that this is the initial value for title
+    # New dropdown options are added to existing field but existing options are not changed
+    return_dict = field.copy()
+    field_id = str(field["id"])
+
+    options = field.get("custom_field_options", [])
+    new_options = {o["value"]: o["name"] for o in options}
+    existing_field = custom_fields_state.get(field_id)
+    if existing_field:
+        existing_options = existing_field["options"]
+        if return_options := return_dict.get("custom_field_options"):
+            for item in return_options:
+                item["name"] = existing_options.get(item["value"], item["name"])
+        for key, value in new_options.items():
+            if key not in existing_options:
+                existing_options[key] = value
+    else:
+        custom_fields_state[field_id] = dict(title=field["title"], options=new_options)
+        return_dict["initial_title"] = field["title"]
+    return return_dict

ingestr/src/zendesk/helpers/credentials.py

@@ -0,0 +1,54 @@
+"""
+This module handles how credentials are read in dlt sources
+"""
+
+from typing import ClassVar, List, Union
+
+import dlt
+from dlt.common.configuration import configspec
+from dlt.common.configuration.specs import CredentialsConfiguration
+from dlt.common.typing import TSecretValue
+
+
+@configspec
+class ZendeskCredentialsBase(CredentialsConfiguration):
+    """
+    The Base version of all the ZendeskCredential classes.
+    """
+
+    subdomain: str = dlt.config.value
+    __config_gen_annotations__: ClassVar[List[str]] = []
+
+
+@configspec
+class ZendeskCredentialsEmailPass(ZendeskCredentialsBase):
+    """
+    This class is used to store credentials for Email + Password Authentication
+    """
+
+    email: str = dlt.config.value
+    password: TSecretValue = dlt.secrets.value
+
+
+@configspec
+class ZendeskCredentialsOAuth(ZendeskCredentialsBase):
+    """
+    This class is used to store credentials for OAuth Token Authentication
+    """
+
+    oauth_token: TSecretValue = dlt.secrets.value
+
+
+@configspec
+class ZendeskCredentialsToken(ZendeskCredentialsBase):
+    """
+    This class is used to store credentials for Token Authentication
+    """
+
+    email: str = dlt.config.value
+    token: TSecretValue = dlt.secrets.value
+
+
+TZendeskCredentials = Union[
+    ZendeskCredentialsEmailPass, ZendeskCredentialsToken, ZendeskCredentialsOAuth
+]