ingestr 0.8.4__py3-none-any.whl → 0.9.0__py3-none-any.whl

ingestr/src/zendesk/helpers/talk_api.py

@@ -0,0 +1,118 @@
+from enum import Enum
+from typing import Any, Dict, Iterator, Optional, Tuple
+
+from dlt.common.typing import DictStrStr, TDataItems, TSecretValue
+from dlt.sources.helpers.requests import client
+
+from .. import settings
+from .credentials import (
+    TZendeskCredentials,
+    ZendeskCredentialsEmailPass,
+    ZendeskCredentialsOAuth,
+    ZendeskCredentialsToken,
+)
+
+
+class PaginationType(Enum):
+    OFFSET = 0
+    CURSOR = 1
+    STREAM = 2
+    START_TIME = 3
+
+
+class ZendeskAPIClient:
+    """
+    API client used to make requests to Zendesk talk, support and chat API
+    """
+
+    subdomain: str = ""
+    url: str = ""
+    headers: Optional[DictStrStr]
+    auth: Optional[Tuple[str, TSecretValue]]
+
+    def __init__(
+        self, credentials: TZendeskCredentials, url_prefix: Optional[str] = None
+    ) -> None:
+        """
+        Initializer for the API client which is then used to make API calls to the ZendeskAPI
+
+        Args:
+            credentials: ZendeskCredentials object which contains the necessary credentials to authenticate to ZendeskAPI
+        """
+        # oauth token is the preferred way to authenticate, followed by api token and then email + password combo
+        # fill headers and auth for every possibility of credentials given, raise error if credentials are of incorrect type
+        if isinstance(credentials, ZendeskCredentialsOAuth):
+            self.headers = {"Authorization": f"Bearer {credentials.oauth_token}"}
+            self.auth = None
+        elif isinstance(credentials, ZendeskCredentialsToken):
+            self.headers = None
+            self.auth = (f"{credentials.email}/token", credentials.token)
+        elif isinstance(credentials, ZendeskCredentialsEmailPass):
+            self.auth = (credentials.email, credentials.password)
+            self.headers = None
+        else:
+            raise TypeError(
+                "Wrong credentials type provided to ZendeskAPIClient. The credentials need to be of type: ZendeskCredentialsOAuth, ZendeskCredentialsToken or ZendeskCredentialsEmailPass"
+            )
+
+        # If url_prefix is set it overrides the default API URL (e.g. chat api uses zopim.com domain)
+        if url_prefix:
+            self.url = url_prefix
+        else:
+            self.subdomain = credentials.subdomain
+            self.url = f"https://{self.subdomain}.zendesk.com"
+
+    def get_pages(
+        self,
+        endpoint: str,
+        data_point_name: str,
+        pagination: PaginationType,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Iterator[TDataItems]:
+        """
+        Makes a request to a paginated endpoint and returns a generator of data items per page.
+
+        Args:
+            endpoint: The url to the endpoint, e.g. /api/v2/calls
+            data_point_name: The key which data items are nested under in the response object (e.g. calls)
+            params: Optional dict of query params to include in the request
+            pagination: Type of pagination type used by endpoint
+
+        Returns:
+            Generator of pages, each page is a list of dict data items
+        """
+        # update the page size to enable cursor pagination
+        params = params or {}
+        if pagination == PaginationType.CURSOR:
+            params["page[size]"] = settings.PAGE_SIZE
+        elif pagination == PaginationType.STREAM:
+            params["per_page"] = settings.INCREMENTAL_PAGE_SIZE
+        elif pagination == PaginationType.START_TIME:
+            params["limit"] = settings.INCREMENTAL_PAGE_SIZE
+
+        # make request and keep looping until there is no next page
+        get_url = f"{self.url}{endpoint}"
+        while get_url:
+            response = client.get(
+                get_url, headers=self.headers, auth=self.auth, params=params
+            )
+            response.raise_for_status()
+            response_json = response.json()
+            result = response_json[data_point_name]
+            yield result
+
+            get_url = None
+            if pagination == PaginationType.CURSOR:
+                if response_json["meta"]["has_more"]:
+                    get_url = response_json["links"]["next"]
+            elif pagination == PaginationType.OFFSET:
+                get_url = response_json.get("next_page", None)
+            elif pagination == PaginationType.STREAM:
+                # See https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#json-format
+                if not response_json["end_of_stream"]:
+                    get_url = response_json["next_page"]
+            elif pagination == PaginationType.START_TIME:
+                if response_json["count"] > 0:
+                    get_url = response_json["next_page"]
+
+            params = {}

ingestr/src/zendesk/settings.py

@@ -0,0 +1,57 @@
+"""Zendesk source settings and constants"""
+
+from dlt.common import pendulum
+
+DEFAULT_START_DATE = pendulum.datetime(year=2024, month=10, day=3)
+
+INCREMENTAL_PAGE_SIZE = 1000
+PAGE_SIZE = 100
+
+
+CUSTOM_FIELDS_STATE_KEY = "ticket_custom_fields_v2"
+
+# Tuples of (Resource name, endpoint URL, data_key, supports pagination)
+# data_key is the key which data list is nested under in responses
+# if the data key is None it is assumed to be the same as the resource name
+# The last element of the tuple says if endpoint supports cursor pagination
+SUPPORT_ENDPOINTS = [
+    ("users", "/api/v2/users.json", "users", True),
+    ("sla_policies", "/api/v2/slas/policies.json", None, False),
+    ("groups", "/api/v2/groups.json", None, True),
+    ("organizations", "/api/v2/organizations.json", None, True),
+    ("brands", "/api/v2/brands.json", None, True),
+]
+
+SUPPORT_EXTRA_ENDPOINTS = [
+    ("activities", "/api/v2/activities.json", None, True),
+    ("automations", "/api/v2/automations.json", None, True),
+    ("macros", "/api/v2/macros.json", None, True),
+    ("recipient_addresses", "/api/v2/recipient_addresses.json", None, True),
+    ("requests", "/api/v2/requests.json", None, True),
+    ("targets", "/api/v2/targets.json", None, False),
+    ("ticket_forms", "/api/v2/ticket_forms.json", None, False),
+    ("ticket_metrics", "/api/v2/ticket_metrics.json", None, True),
+    ("triggers", "/api/v2/triggers.json", None, True),
+    ("user_fields", "/api/v2/user_fields.json", None, True),
+]
+
+TALK_ENDPOINTS = [
+    ("calls", "/api/v2/channels/voice/calls", None, False),
+    ("addresses", "/api/v2/channels/voice/addresses", None, False),
+    ("greetings", "/api/v2/channels/voice/greetings", None, False),
+    ("phone_numbers", "/api/v2/channels/voice/phone_numbers", None, False),
+    ("settings", "/api/v2/channels/voice/settings", None, False),
+    ("lines", "/api/v2/channels/voice/lines", None, False),
+    ("agents_activity", "/api/v2/channels/voice/stats/agents_activity", None, False),
+    (
+        "current_queue_activity",
+        "/api/v2/channels/voice/stats/current_queue_activity",
+        None,
+        False,
+    ),
+]
+
+INCREMENTAL_TALK_ENDPOINTS = {
+    "calls": "/api/v2/channels/voice/stats/incremental/calls.json",
+    "legs": "/api/v2/channels/voice/stats/incremental/legs.json",
+}

{ingestr-0.8.4.dist-info → ingestr-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.8.4
+Version: 0.9.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -35,6 +35,7 @@ Requires-Dist: pyrate-limiter==3.6.1
 Requires-Dist: redshift-connector==2.1.0
 Requires-Dist: rich==13.7.1
 Requires-Dist: rudder-sdk-python==2.1.0
+Requires-Dist: s3fs==2024.9.0
 Requires-Dist: snowflake-sqlalchemy==1.5.3
 Requires-Dist: sqlalchemy-bigquery==1.11.0
 Requires-Dist: sqlalchemy-hana==2.0.0
@@ -226,6 +227,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>Notion</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>S3</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Shopify</td>
@@ -242,6 +248,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Zendesk</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
 </table>
 
 More to come soon!

{ingestr-0.8.4.dist-info → ingestr-0.9.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
 ingestr/main.py,sha256=U66TM57ePv-RdoAftQ0pFZx8woZUQnLepKa50C-bA5I,17655
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/destinations.py,sha256=2SfPMjtTelPmzQmc3zNs8xGcKIPuGn_hoZFIBUuhjXI,6338
-ingestr/src/factory.py,sha256=-_KwBpbNAegv_oXIB57klyjUb3K6e0lw_xdi5bwmarI,4645
-ingestr/src/sources.py,sha256=0IGguMm85E3Rahu6zVLawoe2d4lqRY31uHuxlqCsiQc,25324
+ingestr/src/factory.py,sha256=NbSLbF2ClHRcptpmL0n9p7bGVz8Uj6xyG0Y93OZTfKM,4830
+ingestr/src/sources.py,sha256=Ny7qYulZUmwrCsG8UZ_SeR6FIqVR3E1FWdvj_yQD41M,30786
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
-ingestr/src/version.py,sha256=jhHEJFZWhkQDemoZMomBYq-RNrKXknYzUaeIU9A6XsI,22
+ingestr/src/version.py,sha256=H9NWRZb7NbeRRPLP_V1fARmLNXranorVM-OOY-8_2ug,22
 ingestr/src/adjust/_init_.py,sha256=_jJE3Ywvv-YyJ7ywICdht_X2Gnd1cKm6F1wAfnpXuWM,890
 ingestr/src/adjust/helpers.py,sha256=kkYC3MqMHLNucuQ50klZWrvd3o8VfUysNtZTQSsKZ_c,2588
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
@@ -17,6 +17,9 @@ ingestr/src/facebook_ads/__init__.py,sha256=ZZyogV48gmhDcC3CYQEsC4qT3Q6JI9IOnMff
 ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
 ingestr/src/facebook_ads/helpers.py,sha256=ZLbNHiKer5lPb4g3_435XeBJr57Wv0o1KTyBA1mQ100,9068
 ingestr/src/facebook_ads/settings.py,sha256=1IxZeP_4rN3IBvAncNHOoqpzAirx0Hz-MUK_tl6UTFk,4881
+ingestr/src/filesystem/__init__.py,sha256=wHHaKFuAjsR_ZRjl6g_Flf6FhVs9qhwREthTr03_7cc,4162
+ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
+ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
 ingestr/src/google_sheets/README.md,sha256=wFQhvmGpRA38Ba2N_WIax6duyD4c7c_pwvvprRfQDnw,5470
 ingestr/src/google_sheets/__init__.py,sha256=5qlX-6ilx5MW7klC7B_0jGSxloQSLkSESTh4nlY3Aos,6643
 ingestr/src/google_sheets/helpers/__init__.py,sha256=5hXZrZK8cMO3UOuL-s4OKOpdACdihQD0hYYlSEu-iQ8,35
@@ -39,7 +42,7 @@ ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMO
 ingestr/src/notion/helpers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falPrnKg_y4,5500
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
-ingestr/src/shopify/__init__.py,sha256=UyPquBb9FYww8KMotdJexy8OP69ArnZpbDFoVh-yZeU,62691
+ingestr/src/shopify/__init__.py,sha256=LWO5hfPfwPrq-gZwSRHThDfLHbGo2wkm6CrN1BCm3CQ,62815
 ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
 ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
 ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSsDk,141
@@ -56,6 +59,12 @@ ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu6
 ingestr/src/stripe_analytics/settings.py,sha256=rl9L5XumxO0pjkZf7MGesXHp4QLRgnz3RWLuDWDBKXo,380
 ingestr/src/telemetry/event.py,sha256=MpWc5tt0lSJ1pWKe9HQ11BHrcPBxSH40l4wjZi9u0tI,924
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
+ingestr/src/zendesk/__init__.py,sha256=FjsDhzKaK7RgFs9zIGjMHHl8TlfYRN3TKocbGiJklD4,17570
+ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
+ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
+ingestr/src/zendesk/helpers/api_helpers.py,sha256=dMkNn4ZQXgJTDOXAAXdmRt41phNFoRhYyPaLJih0pZY,4184
+ingestr/src/zendesk/helpers/credentials.py,sha256=EWyi0ZlxWFgd1huD86KNF4dApLHgmabqWksFpEg1cf0,1332
+ingestr/src/zendesk/helpers/talk_api.py,sha256=TSVSOErsBZvxcX91LMhAgvy6yLSYvpuVfOyKViOHtvA,4718
 ingestr/testdata/.gitignore,sha256=DFzYYOpqdTiT7S1HjCT-jffZSmEvFZge295_upAB0FY,13
 ingestr/testdata/create_replace.csv,sha256=TQDbOSkRKq9ZZv1d68Qjwh94aIyUQ-oEwxpJIrd3YK8,1060
 ingestr/testdata/delete_insert_expected.csv,sha256=wbj7uboVWwm3sNMh1n7f4-OKFEQJv1s96snjEHp9nkg,336
@@ -64,8 +73,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.8.4.dist-info/METADATA,sha256=3A0SR09KmNhx0b7jDkrqTM3PwBqtm77dG3XqhCwemHM,6755
-ingestr-0.8.4.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-ingestr-0.8.4.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.8.4.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.8.4.dist-info/RECORD,,
+ingestr-0.9.0.dist-info/METADATA,sha256=yJZ6Tdum4TirrssuFG40UZ_zm4pzhzSSweJRIignFck,6949
+ingestr-0.9.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+ingestr-0.9.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.9.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.9.0.dist-info/RECORD,,