ingestr 0.5.0__tar.gz → 0.6.0__tar.gz

{ingestr-0.5.0 → ingestr-0.6.0}/Dockerfile

@@ -10,7 +10,7 @@ ENV VIRTUAL_ENV=/usr/local
 ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
 RUN /install.sh && rm /install.sh
 
-RUN /root/.cargo/bin/uv pip install --no-cache -r requirements.txt
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
 
 COPY . /app
 

{ingestr-0.5.0 → ingestr-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.5.0
+Version: 0.6.0
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -180,6 +180,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Shopify</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
 </table>
 
 More to come soon!

{ingestr-0.5.0 → ingestr-0.6.0}/README.md

@@ -138,6 +138,11 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Shopify</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
 </table>
 
 More to come soon!

{ingestr-0.5.0 → ingestr-0.6.0}/docs/.vitepress/config.mjs

@@ -70,6 +70,7 @@ export default defineConfig({
             items: [
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
+              { text: "Shopify", link: "/supported-sources/shopify.md" },
             ],
           },
         ],

{ingestr-0.5.0 → ingestr-0.6.0}/docs/supported-sources/overview.md

@@ -89,6 +89,11 @@ ingestr supports the following sources and destinations:
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Shopify</td>
+        <td>✅</td>
+        <td>❌</td>
+    </tr>
 </table>
 
 More to come soon!

ingestr-0.6.0/docs/supported-sources/shopify.md

@@ -0,0 +1,37 @@
+# Shopify
+[Shopify](https://www.shopify.com/) is a comprehensive e-commerce platform that enables individuals and businesses to create online stores.
+
+ingestr supports Shopify as a source.
+
+## URI Format
+The URI format for Shopify is as follows:
+
+```plaintext
+shopify://<shopify store URL>?api_key=token
+```
+
+URI parameters:
+- `shopify store URI`: the URL of the Shopify store you'd like to connect to, e.g. `myawesomestore.myshopify.com`
+- `api_key`: the API key used for authentication with the Shopify API
+
+The URI is used to connect to the Shopify API for extracting data. More details on setting up Shopify integrations can be found [here](https://shopify.dev/docs/admin-api/getting-started).
+
+## Setting up a Shopify Integration
+
+Shopify requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/shopify#setup-guide).
+
+Once you complete the guide, you should have an API key and the store name to connect to. Let's say your API key is `shpkey_12345` and the store you'd like to connect to is `my-store`, here's a sample command that will copy the data from the Shopify store into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'shopify://my-store.myshopify.com?api_key=shpkey_12345' --source-table 'orders' --dest-uri duckdb:///shopify.duckdb --dest-table 'shopify.orders'
+```
+
+The result of this command will be a table in the `shopify.duckdb` database with JSON columns.
+
+## Available Tables
+Shopify source allows ingesting the following sources into separate tables:
+- `orders`
+- `customers`
+- `products`
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.

{ingestr-0.5.0 → ingestr-0.6.0}/docs/supported-sources/snowflake.md

@@ -13,7 +13,7 @@ snowflake://user:password@account/dbname?warehouse=COMPUTE_WH&role=data_scientis
 URI parameters:
 - `user`: the user name to connect to the database
 - `password`: the password for the user
-- `account`: your Snowflake account identifier
+- `account`: your Snowflake account identifier (copying from snowflake interface gives you org_name.account_name, modify the "." to "-" in the ingestr command)
 - `dbname`: the name of the database to connect to
 - `warehouse`: the name of the warehouse to use (optional)
 - `role`: the name of the role to use (optional)

{ingestr-0.5.0 → ingestr-0.6.0}/ingestr/src/destinations.py

@@ -98,7 +98,7 @@ class RedshiftDestination(GenericSqlDestination):
 
 class DuckDBDestination(GenericSqlDestination):
     def dlt_dest(self, uri: str, **kwargs):
-        return dlt.destinations.duckdb(credentials=uri, **kwargs)
+        return dlt.destinations.duckdb(uri, **kwargs)
 
 
 class MsSQLDestination(GenericSqlDestination):

{ingestr-0.5.0 → ingestr-0.6.0}/ingestr/src/factory.py

@@ -19,6 +19,7 @@ from ingestr.src.sources import (
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
+    ShopifySource,
     SqlSource,
 )
 
@@ -91,6 +92,8 @@ class SourceDestinationFactory:
             return NotionSource()
         elif self.source_scheme == "gsheets":
             return GoogleSheetsSource()
+        elif self.source_scheme == "shopify":
+            return ShopifySource()
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

ingestr-0.6.0/ingestr/src/shopify/__init__.py

@@ -0,0 +1,227 @@
+"""Fetches Shopify Orders and Products."""
+
+from typing import Any, Dict, Iterable, Optional
+
+import dlt
+from dlt.common import jsonpath as jp
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import ShopifyApi, ShopifyPartnerApi, TOrderStatus
+from .settings import (
+    DEFAULT_API_VERSION,
+    DEFAULT_ITEMS_PER_PAGE,
+    DEFAULT_PARTNER_API_VERSION,
+    FIRST_DAY_OF_MILLENNIUM,
+)
+
+
+@dlt.source(name="shopify", max_table_nesting=0)
+def shopify_source(
+    private_app_password: str = dlt.secrets.value,
+    api_version: str = DEFAULT_API_VERSION,
+    shop_url: str = dlt.config.value,
+    start_date: TAnyDateTime = FIRST_DAY_OF_MILLENNIUM,
+    end_date: Optional[TAnyDateTime] = None,
+    created_at_min: TAnyDateTime = FIRST_DAY_OF_MILLENNIUM,
+    items_per_page: int = DEFAULT_ITEMS_PER_PAGE,
+    order_status: TOrderStatus = "any",
+) -> Iterable[DltResource]:
+    """
+    The source for the Shopify pipeline. Available resources are products, orders, and customers.
+
+    `start_time` argument can be used on its own or together with `end_time`. When both are provided
+    data is limited to items updated in that time range.
+    The range is "half-open", meaning elements equal and newer than `start_time` and elements older than `end_time` are included.
+    All resources opt-in to use Airflow scheduler if run as Airflow task
+
+    Args:
+        private_app_password: The app password to the app on your shop.
+        api_version: The API version to use (e.g. 2023-01).
+        shop_url: The URL of your shop (e.g. https://my-shop.myshopify.com).
+        items_per_page: The max number of items to fetch per page. Defaults to 250.
+        start_date: Items updated on or after this date are imported. Defaults to 2000-01-01.
+            If end date is not provided, this is used as the initial value for incremental loading and after the initial run, only new data will be retrieved.
+            Accepts any `date`/`datetime` object or a date/datetime string in ISO 8601 format.
+        end_time: The end time of the range for which to load data.
+            Should be used together with `start_date` to limit the data to items updated in that time range.
+            If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+        created_at_min: The minimum creation date of items to import. Items created on or after this date are loaded. Defaults to 2000-01-01.
+        order_status: The order status to filter by. Can be 'open', 'closed', 'cancelled', or 'any'. Defaults to 'any'.
+
+    Returns:
+        Iterable[DltResource]: A list of DltResource objects representing the data resources.
+    """
+
+    # build client
+    client = ShopifyApi(shop_url, private_app_password, api_version)
+
+    start_date_obj = ensure_pendulum_datetime(start_date)
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+    created_at_min_obj = ensure_pendulum_datetime(created_at_min)
+
+    # define resources
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def products(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for products on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of products.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("products", params)
+
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def orders(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+        status: TOrderStatus = order_status,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for orders on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of orders.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            status=status,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("orders", params)
+
+    @dlt.resource(primary_key="id", write_disposition="merge")
+    def customers(
+        updated_at: dlt.sources.incremental[
+            pendulum.DateTime
+        ] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date_obj,
+            end_value=end_date_obj,
+            allow_external_schedulers=True,
+        ),
+        created_at_min: pendulum.DateTime = created_at_min_obj,
+        items_per_page: int = items_per_page,
+    ) -> Iterable[TDataItem]:
+        """
+        The resource for customers on your shop, supports incremental loading and pagination.
+
+        Args:
+            updated_at: The saved state of the last 'updated_at' value.
+
+        Returns:
+            Iterable[TDataItem]: A generator of customers.
+        """
+        params = dict(
+            updated_at_min=updated_at.last_value.isoformat(),
+            limit=items_per_page,
+            order="updated_at asc",
+            created_at_min=created_at_min.isoformat(),
+        )
+        if updated_at.end_value is not None:
+            params["updated_at_max"] = updated_at.end_value.isoformat()
+        yield from client.get_pages("customers", params)
+
+    return (products, orders, customers)
+
+
+@dlt.resource
+def shopify_partner_query(
+    query: str,
+    data_items_path: jp.TJsonPath,
+    pagination_cursor_path: jp.TJsonPath,
+    pagination_variable_name: str = "after",
+    variables: Optional[Dict[str, Any]] = None,
+    access_token: str = dlt.secrets.value,
+    organization_id: str = dlt.config.value,
+    api_version: str = DEFAULT_PARTNER_API_VERSION,
+) -> Iterable[TDataItem]:
+    """
+    Resource for getting paginated results from the Shopify Partner GraphQL API.
+
+    This resource will run the given GraphQL query and extract a list of data items from the result.
+    It will then run the query again with a pagination cursor to get the next page of results.
+
+    Example:
+        query = '''query Transactions($after: String) {
+            transactions(after: $after, first: 100) {
+                edges {
+                    cursor
+                    node {
+                        id
+                    }
+                }
+            }
+        }'''
+
+        partner_query_pages(
+            query,
+            data_items_path="data.transactions.edges[*].node",
+            pagination_cursor_path="data.transactions.edges[-1].cursor",
+            pagination_variable_name="after",
+        )
+
+    Args:
+        query: The GraphQL query to run.
+        data_items_path: The JSONPath to the data items in the query result. Should resolve to array items.
+        pagination_cursor_path: The JSONPath to the pagination cursor in the query result, will be piped to the next query via variables.
+        pagination_variable_name: The name of the variable to pass the pagination cursor to.
+        variables: Mapping of extra variables used in the query.
+        access_token: The Partner API Client access token, created in the Partner Dashboard.
+        organization_id: Your Organization ID, found in the Partner Dashboard.
+        api_version: The API version to use (e.g. 2024-01). Use `unstable` for the latest version.
+    Returns:
+        Iterable[TDataItem]: A generator of the query results.
+    """
+    client = ShopifyPartnerApi(
+        access_token=access_token,
+        organization_id=organization_id,
+        api_version=api_version,
+    )
+
+    yield from client.get_graphql_pages(
+        query,
+        data_items_path=data_items_path,
+        pagination_cursor_path=pagination_cursor_path,
+        pagination_variable_name=pagination_variable_name,
+        variables=variables,
+    )

ingestr-0.6.0/ingestr/src/shopify/exceptions.py

@@ -0,0 +1,2 @@
+class ShopifyPartnerApiError(Exception):
+    pass

ingestr-0.6.0/ingestr/src/shopify/helpers.py

@@ -0,0 +1,147 @@
+"""Shopify source helpers"""
+
+from typing import Any, Iterable, Literal, Optional
+from urllib.parse import urljoin
+
+from dlt.common import jsonpath
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import Dict, DictStrAny, TDataItems
+from dlt.sources.helpers import requests
+
+from .exceptions import ShopifyPartnerApiError
+from .settings import DEFAULT_API_VERSION, DEFAULT_PARTNER_API_VERSION
+
+TOrderStatus = Literal["open", "closed", "cancelled", "any"]
+
+
+class ShopifyApi:
+    """
+    A Shopify API client that can be used to get pages of data from Shopify.
+    """
+
+    def __init__(
+        self,
+        shop_url: str,
+        private_app_password: str,
+        api_version: str = DEFAULT_API_VERSION,
+    ) -> None:
+        """
+        Args:
+            shop_url: The URL of your shop (e.g. https://my-shop.myshopify.com).
+            private_app_password: The private app password to the app on your shop.
+            api_version: The API version to use (e.g. 2023-01)
+        """
+        self.shop_url = shop_url
+        self.private_app_password = private_app_password
+        self.api_version = api_version
+
+    def get_pages(
+        self, resource: str, params: Optional[Dict[str, Any]] = None
+    ) -> Iterable[TDataItems]:
+        """Get all pages from shopify using requests.
+        Iterates through all pages and yield each page items.
+
+        Args:
+            resource: The resource to get pages for (e.g. products, orders, customers).
+            params: Query params to include in the request.
+
+        Yields:
+            List of data items from the page
+        """
+        url = urljoin(self.shop_url, f"/admin/api/{self.api_version}/{resource}.json")
+
+        headers = {"X-Shopify-Access-Token": self.private_app_password}
+        while url:
+            response = requests.get(url, params=params, headers=headers)
+            response.raise_for_status()
+            json = response.json()
+            # Get item list from the page
+            yield [self._convert_datetime_fields(item) for item in json[resource]]
+            url = response.links.get("next", {}).get("url")
+            # Query params are included in subsequent page URLs
+            params = None
+
+    def _convert_datetime_fields(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """Convert timestamp fields in the item to pendulum datetime objects
+
+        The item is modified in place.
+
+        Args:
+            item: The item to convert
+
+        Returns:
+            The same data item (for convenience)
+        """
+        fields = ["created_at", "updated_at"]
+        for field in fields:
+            if field in item:
+                item[field] = ensure_pendulum_datetime(item[field])
+        return item
+
+
+class ShopifyPartnerApi:
+    """Client for Shopify Partner grapql API"""
+
+    def __init__(
+        self,
+        access_token: str,
+        organization_id: str,
+        api_version: str = DEFAULT_PARTNER_API_VERSION,
+    ) -> None:
+        """
+        Args:
+            access_token: The access token to use
+            organization_id: The organization id to query
+            api_version: The API version to use (e.g. 2023-01)
+        """
+        self.access_token = access_token
+        self.organization_id = organization_id
+        self.api_version = api_version
+
+    @property
+    def graphql_url(self) -> str:
+        return f"https://partners.shopify.com/{self.organization_id}/api/{self.api_version}/graphql.json"
+
+    def run_graphql_query(
+        self, query: str, variables: Optional[DictStrAny] = None
+    ) -> DictStrAny:
+        """Run a graphql query against the Shopify Partner API
+
+        Args:
+            query: The query to run
+            variables: The variables to include in the query
+
+        Returns:
+            The response JSON
+        """
+        headers = {"X-Shopify-Access-Token": self.access_token}
+        response = requests.post(
+            self.graphql_url,
+            json={"query": query, "variables": variables},
+            headers=headers,
+        )
+        data = response.json()
+        if data.get("errors"):
+            raise ShopifyPartnerApiError(response.text)
+        return data  # type: ignore[no-any-return]
+
+    def get_graphql_pages(
+        self,
+        query: str,
+        data_items_path: jsonpath.TJsonPath,
+        pagination_cursor_path: jsonpath.TJsonPath,
+        pagination_variable_name: str,
+        variables: Optional[DictStrAny] = None,
+    ) -> Iterable[TDataItems]:
+        variables = dict(variables or {})
+        while True:
+            data = self.run_graphql_query(query, variables)
+            print(data)
+            data_items = jsonpath.find_values(data_items_path, data)
+            if not data_items:
+                break
+            yield data_items
+            cursors = jsonpath.find_values(pagination_cursor_path, data)
+            if not cursors:
+                break
+            variables[pagination_variable_name] = cursors[-1]

ingestr-0.6.0/ingestr/src/shopify/settings.py

@@ -0,0 +1,5 @@
+FIRST_DAY_OF_MILLENNIUM = "2000-01-01"
+DEFAULT_API_VERSION = "2023-10"
+DEFAULT_ITEMS_PER_PAGE = 250
+
+DEFAULT_PARTNER_API_VERSION = "2024-01"

{ingestr-0.5.0 → ingestr-0.6.0}/ingestr/src/sources.py

@@ -9,6 +9,7 @@ import dlt
 from ingestr.src.google_sheets import google_spreadsheet
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
+from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
 
 
@@ -134,6 +135,43 @@ class NotionSource:
         )
 
 
+class ShopifySource:
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Shopify takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        # shopify://shop_url?api_key=private_app_password
+
+        source_fields = urlparse(uri)
+        source_params = parse_qs(source_fields.query)
+        api_key = source_params.get("api_key")
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Shopify")
+
+        date_args = {}
+        if kwargs.get("interval_start"):
+            date_args["start_date"] = kwargs.get("interval_start")
+
+        if kwargs.get("interval_end"):
+            date_args["end_date"] = kwargs.get("interval_end")
+
+        resource = None
+        if table in ["products", "orders", "customers"]:
+            resource = table
+        else:
+            raise ValueError(
+                f"Table name '{table}' is not supported for Shopify source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        return shopify_source(
+            private_app_password=api_key[0],
+            shop_url=f"https://{source_fields.netloc}",
+            **date_args,
+        ).with_resources(resource)
+
+
 class GoogleSheetsSource:
     table_builder: Callable
 

ingestr-0.6.0/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

{ingestr-0.5.0 → ingestr-0.6.0}/pyproject.toml

@@ -66,6 +66,7 @@ exclude = [
     'src/sql_database/.*',
     'src/mongodb/.*',
     'src/google_sheets/.*',
+    'src/shopify/.*',
 ]
 
 [[tool.mypy.overrides]]
@@ -73,6 +74,7 @@ module = [
   "ingestr.src.sql_database.*",
   "ingestr.src.mongodb.*",
   "ingestr.src.google_sheets.*",
+  "ingestr.src.shopify.*",
 ]
 follow_imports = "skip"
 

ingestr-0.5.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.5.0"