ingestr 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

ingestr/src/gorgias/helpers.py

@@ -0,0 +1,149 @@
+"""Gorgias source helpers"""
+
+from typing import Any, Iterable, Optional, Tuple
+
+from dlt.common.pendulum import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import Dict, TDataItems
+from dlt.sources.helpers import requests
+from pyrate_limiter import Duration, Limiter, Rate
+from requests.auth import HTTPBasicAuth
+
+
+def get_max_datetime_from_datetime_fields(
+    item: Dict[str, Any],
+) -> Tuple[str, Optional[pendulum.DateTime]]:
+    """Get the maximum datetime from any field that ends with _datetime"""
+
+    max_field_name = None
+    max_field_value = None
+    for field in item:
+        if field.endswith("_datetime") and item[field] is not None:
+            dt = ensure_pendulum_datetime(item[field])
+            if not max_field_name or dt > max_field_value:
+                max_field_name = field
+                max_field_value = dt
+
+    return max_field_name, max_field_value
+
+
+def convert_datetime_fields(item: Dict[str, Any]) -> Dict[str, Any]:
+    for field in item:
+        if field.endswith("_datetime") and item[field] is not None:
+            item[field] = ensure_pendulum_datetime(item[field])
+
+    if "updated_datetime" not in item:
+        _, max_datetime = get_max_datetime_from_datetime_fields(item)
+        item["updated_datetime"] = max_datetime
+
+    return item
+
+
+def find_latest_timestamp_from_page(
+    items: list[Dict[str, Any]],
+) -> Optional[Dict[str, Any]]:
+    latest_time = None
+    for item in items:
+        _, max_field_value = get_max_datetime_from_datetime_fields(item)
+        if not latest_time or ensure_pendulum_datetime(max_field_value) > latest_time:
+            latest_time = max_field_value
+
+    return latest_time
+
+
+class GorgiasApi:
+    """
+    A Gorgias API client that can be used to get pages of data from Gorgias.
+    """
+
+    def __init__(
+        self,
+        domain: str,
+        email: str,
+        api_key: str,
+    ) -> None:
+        """
+        Args:
+            domain: The domain of your Gorgias account.
+            email: The email associated with your Gorgias account.
+            api_key: The API key for accessing the Gorgias API.
+        """
+        self.domain = domain
+        self.email = email
+        self.api_key = api_key
+
+    def get_pages(
+        self,
+        resource: str,
+        params: Optional[Dict[str, Any]] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+    ) -> Iterable[TDataItems]:
+        """Get all pages from Gorgias using requests.
+        Iterates through all pages and yield each page items.
+
+        Args:
+            resource: The resource to get pages for (e.g. products, orders, customers).
+            params: Query params to include in the request.
+
+        Yields:
+            List of data items from the page
+        """
+        url = f"https://{self.domain}.gorgias.com/api/{resource}"
+        rate = Rate(2, Duration.SECOND)
+        limiter = Limiter(rate, raise_when_fail=False)
+
+        start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
+
+        if not params:
+            params = {}
+
+        params["limit"] = 100
+        if "order_by" not in params:
+            params["order_by"] = "updated_datetime:desc"
+
+        while True:
+            limiter.try_acquire(f"gorgias-{self.domain}")
+            response = requests.get(
+                url, params=params, auth=HTTPBasicAuth(self.email, self.api_key)
+            )
+            response.raise_for_status()
+            if len(response.json()["data"]) == 0:
+                break
+
+            json = response.json()
+
+            items = self.__filter_items_in_range(json["data"], start_date, end_date)
+            if len(items) > 0:
+                yield items
+
+            # if there is no cursor, yield the items first and then break the loop
+            cursor = json.get("meta", {}).get("next_cursor")
+            params["cursor"] = cursor
+            if not cursor:
+                break
+
+            if start_date_obj:
+                max_datetime = find_latest_timestamp_from_page(json["data"])
+                if start_date_obj > ensure_pendulum_datetime(max_datetime):
+                    break
+
+    def __filter_items_in_range(
+        self,
+        items: list[Dict[str, Any]],
+        start_date: Optional[str],
+        end_date: Optional[str],
+    ) -> list[Dict[str, Any]]:
+        start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
+        end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+        filtered = []
+        for item in items:
+            converted_item = convert_datetime_fields(item)
+            if start_date_obj and item["updated_datetime"] < start_date_obj:
+                continue
+            if end_date_obj and item["updated_datetime"] > end_date_obj:
+                continue
+            filtered.append(converted_item)
+
+        return filtered

ingestr/src/gorgias/helpers_test.py

@@ -0,0 +1,45 @@
+from dlt.common.pendulum import pendulum
+
+from .helpers import convert_datetime_fields, find_latest_timestamp_from_page
+
+
+def test_convert_datetime_fields():
+    item = {
+        "key1": "val1",
+        "created_datetime": "2024-06-20T07:39:36.514848+00:00",
+        "sent_datetime": "2024-06-20T07:40:20.166593+00:00",
+        "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
+    }
+
+    actual = convert_datetime_fields(item)
+
+    assert actual == {
+        "key1": "val1",
+        "created_datetime": pendulum.datetime(2024, 6, 20, 7, 39, 36, 514848, tz="UTC"),
+        "sent_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
+        "should_send_datetime": pendulum.datetime(
+            2024, 6, 20, 7, 39, 37, 514848, tz="UTC"
+        ),
+        "updated_datetime": pendulum.datetime(2024, 6, 20, 7, 40, 20, 166593, tz="UTC"),
+    }
+
+
+def test_find_latest_timestamp_from_page():
+    items = [
+        {
+            "key1": "val1",
+            "created_datetime": "2024-06-20T07:39:36.514848+00:00",
+            "sent_datetime": "2024-06-20T07:40:20.166593+00:00",
+            "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
+        },
+        {
+            "key1": "val2",
+            "created_datetime": "2024-06-20T07:39:36.514848+00:00",
+            "sent_datetime": "2024-06-20T07:40:21.123123+00:00",
+            "should_send_datetime": "2024-06-20T07:39:37.514848+00:00",
+        },
+    ]
+
+    actual = find_latest_timestamp_from_page(items)
+
+    assert actual == pendulum.datetime(2024, 6, 20, 7, 40, 21, 123123, tz="UTC")

ingestr/src/shopify/__init__.py

@@ -0,0 +1,227 @@
+"""Fetches Shopify Orders and Products."""
+
+from typing import Any, Dict, Iterable, Optional
+
+import dlt
+from dlt.common import jsonpath as jp
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import ShopifyApi, ShopifyPartnerApi, TOrderStatus
+from .settings import (
+    DEFAULT_API_VERSION,
+    DEFAULT_ITEMS_PER_PAGE,
+    DEFAULT_PARTNER_API_VERSION,
+    FIRST_DAY_OF_MILLENNIUM,
+)
+
+
+@dlt.source(name="shopify", max_table_nesting=0)
+def shopify_source(
+    private_app_password: str = dlt.secrets.value,
+    api_version: str = DEFAULT_API_VERSION,
+    shop_url: str = dlt.config.value,
+    start_date: TAnyDateTime = FIRST_DAY_OF_MILLENNIUM,
+    end_date: Optional[TAnyDateTime] = None,
+    created_at_min: TAnyDateTime = FIRST_DAY_OF_MILLENNIUM,
+    items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
+    order_status: TOrderStatus = "any",
+) -> Iterable[DltResource]:
+    """
+    The source for the Shopify pipeline. Available resources are products, orders, and customers.
+
+    `start_time` argument can be used on its own or together with `end_time`. When both are provided
+    data is limited to items updated in that time range.
+    The range is "half-open", meaning elements equal and newer than `start_time` and elements older than `end_time` are included.
+    All resources opt-in to use Airflow scheduler if run as Airflow task
+
+    Args:
+        private_app_password: The app password to the app on your shop.
+        api_version: The API version to use (e.g. 2023-01).
+        shop_url: The URL of your shop (e.g. https://my-shop.myshopify.com).
+        items_per_page: The max number of items to fetch per page. Defaults to 250.
+        start_date: Items updated on or after this date are imported. Defaults to 2000-01-01.
+            If end date is not provided, this is used as the initial value for incremental loading and after the initial run, only new data will be retrieved.
+            Accepts any `date`/`datetime` object or a date/datetime string in ISO 8601 format.
+        end_time: The end time of the range for which to load data.
+            Should be used together with `start_date` to limit the data to items updated in that time range.
+            If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+        created_at_min: The minimum creation date of items to import. Items created on or after this date are loaded. Defaults to 2000-01-01.
+        order_status: The order status to filter by. Can be 'open', 'closed', 'cancelled', or 'any'. Defaults to 'any'.
+
+    Returns:
+        Iterable[DltResource]: A list of DltResource objects representing the data resources.
+    """
+
+    # build client
+    client = ShopifyApi(shop_url, private_app_password, api_version)
+
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+    created_at_min_obj = ensure_pendulum_datetime(created_at_min)
+
+    # define resources
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def products(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for products on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of products.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("products", params)
+
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def orders(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+        status: TOrderStatus = order_status,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for orders on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of orders.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            status=status,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("orders", params)
+
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def customers(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for customers on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of customers.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("customers", params)
+
+    return (products, orders, customers)
+
+
+@dlt.resource
+def shopify_partner_query(
+    query: str,
+    data_items_path: jp.TJsonPath,
+    pagination_cursor_path: jp.TJsonPath,
+    pagination_variable_name: str = "after",
+    variables: Optional[Dict[str, Any]] = None,
+    access_token: str = dlt.secrets.value,
+    organization_id: str = dlt.config.value,
+    api_version: str = DEFAULT_PARTNER_API_VERSION,
+) -> Iterable[TDataItem]:
+    """
+    Resource for getting paginated results from the Shopify Partner GraphQL API.
+
+    This resource will run the given GraphQL query and extract a list of data items from the result.
+    It will then run the query again with a pagination cursor to get the next page of results.
+
+    Example:
+        query = '''query Transactions($after: String) {
+            transactions(after: $after, first: 100) {
+                edges {
+                    cursor
+                    node {
+                        id
+                    }
+                }
+            }
+        }'''
+
+        partner_query_pages(
+            query,
+            data_items_path="data.transactions.edges[*].node",
+            pagination_cursor_path="data.transactions.edges[-1].cursor",
+            pagination_variable_name="after",
+        )
+
+    Args:
+        query: The GraphQL query to run.
+        data_items_path: The JSONPath to the data items in the query result. Should resolve to array items.
+        pagination_cursor_path: The JSONPath to the pagination cursor in the query result, will be piped to the next query via variables.
+        pagination_variable_name: The name of the variable to pass the pagination cursor to.
+        variables: Mapping of extra variables used in the query.
+        access_token: The Partner API Client access token, created in the Partner Dashboard.
+        organization_id: Your Organization ID, found in the Partner Dashboard.
+        api_version: The API version to use (e.g. 2024-01). Use `unstable` for the latest version.
+    Returns:
+        Iterable[TDataItem]: A generator of the query results.
+    """
+    client = ShopifyPartnerApi(
+        access_token=access_token,
+        organization_id=organization_id,
+        api_version=api_version,
+    )
+
+    yield from client.get_graphql_pages(
+        query,
+        data_items_path=data_items_path,
+        pagination_cursor_path=pagination_cursor_path,
+        pagination_variable_name=pagination_variable_name,
+        variables=variables,
+    )

ingestr/src/shopify/exceptions.py

@@ -0,0 +1,2 @@
+class ShopifyPartnerApiError(Exception):
+    pass

ingestr/src/shopify/helpers.py

@@ -0,0 +1,147 @@
+"""Shopify source helpers"""
+
+from typing import Any, Iterable, Literal, Optional
+from urllib.parse import urljoin
+
+from dlt.common import jsonpath
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import Dict, DictStrAny, TDataItems
+from dlt.sources.helpers import requests
+
+from .exceptions import ShopifyPartnerApiError
+from .settings import DEFAULT_API_VERSION, DEFAULT_PARTNER_API_VERSION
+
+TOrderStatus = Literal["open", "closed", "cancelled", "any"]
+
+
+class ShopifyApi:
+    """
+    A Shopify API client that can be used to get pages of data from Shopify.
+    """
+
+    def __init__(
+        self,
+        shop_url: str,
+        private_app_password: str,
+        api_version: str = DEFAULT_API_VERSION,
+    ) -> None:
+        """
+        Args:
+            shop_url: The URL of your shop (e.g. https://my-shop.myshopify.com).
+            private_app_password: The private app password to the app on your shop.
+            api_version: The API version to use (e.g. 2023-01)
+        """
+        self.shop_url = shop_url
+        self.private_app_password = private_app_password
+        self.api_version = api_version
+
+    def get_pages(
+        self, resource: str, params: Optional[Dict[str, Any]] = None
+    ) -> Iterable[TDataItems]:
+        """Get all pages from shopify using requests.
+        Iterates through all pages and yield each page items.
+
+        Args:
+            resource: The resource to get pages for (e.g. products, orders, customers).
+            params: Query params to include in the request.
+
+        Yields:
+            List of data items from the page
+        """
+        url = urljoin(self.shop_url, f"/admin/api/{self.api_version}/{resource}.json")
+
+        headers = {"X-Shopify-Access-Token": self.private_app_password}
+        while url:
+            response = requests.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            json = response.json()
+            # Get item list from the page
+            yield [self._convert_datetime_fields(item) for item in json[resource]]
+            url = response.links.get("next", {}).get("url")
+            # Query params are included in subsequent page URLs
+            params = None
+
+    def _convert_datetime_fields(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert timestamp fields in the item to pendulum datetime objects
+
+        The item is modified in place.
+
+        Args:
+            item: The item to convert
+
+        Returns:
+            The same data item (for convenience)
+        """
+        fields = ["created_at", "updated_at"]
+        for field in fields:
+            if field in item:
+                item[field] = ensure_pendulum_datetime(item[field])
+        return item
+
+
+class ShopifyPartnerApi:
+    """Client for Shopify Partner grapql API"""
+
+    def __init__(
+        self,
+        access_token: str,
+        organization_id: str,
+        api_version: str = DEFAULT_PARTNER_API_VERSION,
+    ) -> None:
+        """
+        Args:
+            access_token: The access token to use
+            organization_id: The organization id to query
+            api_version: The API version to use (e.g. 2023-01)
+        """
+        self.access_token = access_token
+        self.organization_id = organization_id
+        self.api_version = api_version
+
+    @property
+    def graphql_url(self) -> str:
+        return f"https://partners.shopify.com/{self.organization_id}/api/{self.api_version}/graphql.json"
+
+    def run_graphql_query(
+        self, query: str, variables: Optional[DictStrAny] = None
+    ) -> DictStrAny:
+        """Run a graphql query against the Shopify Partner API
+
+        Args:
+            query: The query to run
+            variables: The variables to include in the query
+
+        Returns:
+            The response JSON
+        """
+        headers = {"X-Shopify-Access-Token": self.access_token}
+        response = requests.post(
+            self.graphql_url,
+            json={"query": query, "variables": variables},
+            headers=headers,
+        )
+        data = response.json()
+        if data.get("errors"):
+            raise ShopifyPartnerApiError(response.text)
+        return data  # type: ignore[no-any-return]
+
+    def get_graphql_pages(
+        self,
+        query: str,
+        data_items_path: jsonpath.TJsonPath,
+        pagination_cursor_path: jsonpath.TJsonPath,
+        pagination_variable_name: str,
+        variables: Optional[DictStrAny] = None,
+    ) -> Iterable[TDataItems]:
+        variables = dict(variables or {})
+        while True:
+            data = self.run_graphql_query(query, variables)
+            print(data)
+            data_items = jsonpath.find_values(data_items_path, data)
+            if not data_items:
+                break
+            yield data_items
+            cursors = jsonpath.find_values(pagination_cursor_path, data)
+            if not cursors:
+                break
+            variables[pagination_variable_name] = cursors[-1]

ingestr/src/shopify/settings.py

@@ -0,0 +1,5 @@
+FIRST_DAY_OF_MILLENNIUM = "2000-01-01"
+DEFAULT_API_VERSION = "2023-10"
+DEFAULT_ITEMS_PER_PAGE = 250
+
+DEFAULT_PARTNER_API_VERSION = "2024-01"