ingestr 0.7.3__tar.gz → 0.7.5__tar.gz

{ingestr-0.7.3 → ingestr-0.7.5}/.github/workflows/tests.yml

@@ -72,7 +72,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64,linux/arm64/v8
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

ingestr-0.7.5/Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+
+# Setup dependencies for pyodbc
+RUN \
+  export ACCEPT_EULA='Y' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https
+
+RUN \ 
+  # Install pyodbc db drivers for MSSQL and PostgreSQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
+  
+RUN \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini
+
+ENV VIRTUAL_ENV=/usr/local
+ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
+RUN /install.sh && rm /install.sh
+
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
+
+COPY . /app
+
+RUN pip3 install -e .
+
+ENTRYPOINT ["ingestr"]

{ingestr-0.7.3 → ingestr-0.7.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.3
+Version: 0.7.5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,7 +14,6 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
-Requires-Dist: connectorx==0.3.3
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1
@@ -39,6 +38,7 @@ Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy2-stubs==0.0.2a38
 Requires-Dist: sqlalchemy==1.4.52
+Requires-Dist: stripe==10.7.0
 Requires-Dist: tqdm==4.66.2
 Requires-Dist: typer==0.12.3
 Description-Content-Type: text/markdown
@@ -176,22 +176,27 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Stripe</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
 </table>
 

{ingestr-0.7.3 → ingestr-0.7.5}/README.md

@@ -131,22 +131,27 @@ Join our Slack community [here](https://join.slack.com/t/bruindatacommunity/shar
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Stripe</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
 </table>
 

{ingestr-0.7.3 → ingestr-0.7.5}/docs/.vitepress/config.mjs

@@ -73,6 +73,7 @@ export default defineConfig({
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
+              { text: "Stripe", link: "/supported-sources/stripe.md" },
             ],
           },
         ],

{ingestr-0.7.3 → ingestr-0.7.5}/docs/supported-sources/overview.md

@@ -82,22 +82,27 @@ ingestr supports the following sources and destinations:
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Google Sheets</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
     </tr>
     <tr>
         <td>Shopify</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Stripe</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
 </table>
 

ingestr-0.7.5/docs/supported-sources/stripe.md

@@ -0,0 +1,45 @@
+# Stripe
+[Stripe](https://www.stripe.com/) is a technology company that builds economic infrastructure for the internet, providing payment processing software and APIs for e-commerce websites and mobile applications.
+
+ingestr supports Stripe as a source.
+
+## URI Format
+The URI format for Stripe is as follows:
+
+```plaintext
+stripe://?api_key=<api-key-here>
+```
+
+URI parameters:
+- `api_key`: the API key used for authentication with the Stripe API
+
+The URI is used to connect to the Stripe API for extracting data. More details on setting up Stripe integrations can be found [here](https://stripe.com/docs/api).
+
+## Setting up a Stripe Integration
+
+Stripe requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/stripe#setup-guide).
+
+Once you complete the guide, you should have an API key. Let's say your API key is `sk_test_12345`, here's a sample command that will copy the data from Stripe into a duckdb database:
+
+```sh
+ingestr ingest --source-uri 'stripe://?api_key=sk_test_12345' --source-table 'charges' --dest-uri duckdb:///stripe.duckdb --dest-table 'stripe.charges'
+```
+
+The result of this command will be a table in the `stripe.duckdb` database with JSON columns.
+
+## Available Tables
+Stripe source allows ingesting the following sources into separate tables:
+- `subscription`: Represents a customer's subscription to a recurring service, detailing billing cycles, plans, and status.
+- `account`: Contains information about a Stripe account, including balances, payouts, and account settings.
+- `coupon`: Stores data about discount codes or coupons that can be applied to invoices, subscriptions, or other charges.
+- `customer`: Holds information about customers, such as billing details, payment methods, and associated transactions.
+- `product`: Represents products that can be sold or subscribed to, including metadata and pricing information.
+- `price`: Contains pricing information for products, including currency, amount, and billing intervals.
+- `balancetransaction`: Records transactions that affect the Stripe account balance, such as charges, refunds, and payouts.
+- `invoice`: Represents invoices sent to customers, detailing line items, amounts, and payment status.
+- `event`: Logs all events in the Stripe account, including customer actions, account updates, and system-generated events.
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!WARNING]
+> Stripe does not support incremental loading for many endpoints in its APIs, which means ingestr will load endpoints incrementally if they support it, and do a full-refresh if not.

{ingestr-0.7.3 → ingestr-0.7.5}/ingestr/src/factory.py

@@ -22,6 +22,7 @@ from ingestr.src.sources import (
     NotionSource,
     ShopifySource,
     SqlSource,
+    StripeAnalyticsSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -102,6 +103,9 @@ class SourceDestinationFactory:
             return ShopifySource()
         elif self.source_scheme == "gorgias":
             return GorgiasSource()
+        elif self.source_scheme == "stripe":
+            return StripeAnalyticsSource()
+
         else:
             raise ValueError(f"Unsupported source scheme: {self.source_scheme}")
 

{ingestr-0.7.3 → ingestr-0.7.5}/ingestr/src/sources.py

@@ -12,6 +12,7 @@ from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.sql_database import sql_table
+from ingestr.src.stripe_analytics import stripe_source
 from ingestr.src.table_definition import table_string_to_dataclass
 
 
@@ -295,3 +296,57 @@ class GoogleSheetsSource:
             range_names=[table_fields.dataset],
             get_named_ranges=False,
         )
+
+
+class StripeAnalyticsSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Stripe takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        api_key = None
+        source_field = urlparse(uri)
+        source_params = parse_qs(source_field.query)
+        api_key = source_params.get("api_key")
+
+        if not api_key:
+            raise ValueError("api_key in the URI is required to connect to Stripe")
+
+        endpoint = None
+        table = str.capitalize(table)
+
+        if table in [
+            "Subscription",
+            "Account",
+            "Coupon",
+            "Customer",
+            "Product",
+            "Price",
+            "BalanceTransaction",
+            "Invoice",
+            "Event",
+        ]:
+            endpoint = table
+        else:
+            raise ValueError(
+                f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        date_args = {}
+        if kwargs.get("interval_start"):
+            date_args["start_date"] = kwargs.get("interval_start")
+
+        if kwargs.get("interval_end"):
+            date_args["end_date"] = kwargs.get("interval_end")
+
+        return stripe_source(
+            endpoints=[
+                endpoint,
+            ],
+            stripe_secret_key=api_key[0],
+            **date_args,
+        ).with_resources(endpoint)

ingestr-0.7.5/ingestr/src/stripe_analytics/__init__.py

@@ -0,0 +1,99 @@
+"""This source uses Stripe API and dlt to load data such as Customer, Subscription, Event etc. to the database and to calculate the MRR and churn rate."""
+
+from typing import Any, Dict, Generator, Iterable, Optional, Tuple
+
+import dlt
+import stripe
+from dlt.sources import DltResource
+from pendulum import DateTime
+
+from .helpers import pagination, transform_date
+from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
+
+
+@dlt.source
+def stripe_source(
+    endpoints: Tuple[str, ...] = ENDPOINTS,
+    stripe_secret_key: str = dlt.secrets.value,
+    start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from the Stripe API for the specified endpoints.
+
+    For all endpoints, Stripe API responses do not provide the key "updated",
+    so in most cases, we are forced to load the data in 'replace' mode.
+    This source is suitable for all types of endpoints, including 'Events', 'Invoice', etc.
+    but these endpoints can also be loaded in incremental mode (see source incremental_stripe_source).
+
+    Args:
+        endpoints (Tuple[str, ...]): A tuple of endpoint names to retrieve data from. Defaults to most popular Stripe API endpoints.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date (Optional[DateTime]): An optional start date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to None.
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to None.
+
+    Returns:
+        Iterable[DltResource]: Resources with data that was created during the period greater than or equal to 'start_date' and less than 'end_date'.
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+
+    def stripe_resource(
+        endpoint: str,
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        yield from pagination(endpoint, start_date, end_date)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            stripe_resource,
+            name=endpoint,
+            write_disposition="replace",
+        )(endpoint)
+
+
+@dlt.source
+def incremental_stripe_source(
+    endpoints: Tuple[str, ...] = INCREMENTAL_ENDPOINTS,
+    stripe_secret_key: str = dlt.secrets.value,
+    initial_start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+) -> Iterable[DltResource]:
+    """
+    As Stripe API does not include the "updated" key in its responses,
+    we are only able to perform incremental downloads from endpoints where all objects are uneditable.
+    This source yields the resources with incremental loading based on "append" mode.
+    You will load only the newest data without duplicating and without downloading a huge amount of data each time.
+
+    Args:
+        endpoints (tuple): A tuple of endpoint names to retrieve data from. Defaults to Stripe API endpoints with uneditable data.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        initial_start_date (Optional[DateTime]): An optional parameter that specifies the initial value for dlt.sources.incremental.
+                            If parameter is not None, then load only data that were created after initial_start_date on the first run.
+                            Defaults to None. Format: datetime(YYYY, MM, DD).
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved.
+                  Defaults to None. Format: datetime(YYYY, MM, DD).
+    Returns:
+        Iterable[DltResource]: Resources with only that data has not yet been loaded.
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+    start_date_unix = (
+        transform_date(initial_start_date) if initial_start_date is not None else -1
+    )
+
+    def incremental_resource(
+        endpoint: str,
+        created: Optional[Any] = dlt.sources.incremental(
+            "created", initial_value=start_date_unix
+        ),
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        start_value = created.last_value
+        yield from pagination(endpoint, start_date=start_value, end_date=end_date)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            incremental_resource,
+            name=endpoint,
+            write_disposition="append",
+            primary_key="id",
+        )(endpoint)

ingestr-0.7.5/ingestr/src/stripe_analytics/helpers.py

@@ -0,0 +1,68 @@
+"""Stripe analytics source helpers"""
+
+from typing import Any, Dict, Iterable, Optional, Union
+
+import stripe
+from dlt.common import pendulum
+from dlt.common.typing import TDataItem
+from pendulum import DateTime
+
+
+def pagination(
+    endpoint: str, start_date: Optional[Any] = None, end_date: Optional[Any] = None
+) -> Iterable[TDataItem]:
+    """
+    Retrieves data from an endpoint with pagination.
+
+    Args:
+        endpoint (str): The endpoint to retrieve data from.
+        start_date (Optional[Any]): An optional start date to limit the data retrieved. Defaults to None.
+        end_date (Optional[Any]): An optional end date to limit the data retrieved. Defaults to None.
+
+    Returns:
+        Iterable[TDataItem]: Data items retrieved from the endpoint.
+    """
+    starting_after = None
+    while True:
+        response = stripe_get_data(
+            endpoint,
+            start_date=start_date,
+            end_date=end_date,
+            starting_after=starting_after,
+        )
+
+        if len(response["data"]) > 0:
+            starting_after = response["data"][-1]["id"]
+        yield response["data"]
+
+        if not response["has_more"]:
+            break
+
+
+def transform_date(date: Union[str, DateTime, int]) -> int:
+    if isinstance(date, str):
+        date = pendulum.from_format(date, "%Y-%m-%dT%H:%M:%SZ")
+    if isinstance(date, DateTime):
+        # convert to unix timestamp
+        date = int(date.timestamp())
+    return date
+
+
+def stripe_get_data(
+    resource: str,
+    start_date: Optional[Any] = None,
+    end_date: Optional[Any] = None,
+    **kwargs: Any,
+) -> Dict[Any, Any]:
+    if start_date:
+        start_date = transform_date(start_date)
+    if end_date:
+        end_date = transform_date(end_date)
+
+    if resource == "Subscription":
+        kwargs.update({"status": "all"})
+
+    resource_dict = getattr(stripe, resource).list(
+        created={"gte": start_date, "lt": end_date}, limit=100, **kwargs
+    )
+    return dict(resource_dict)

ingestr-0.7.5/ingestr/src/stripe_analytics/settings.py

@@ -0,0 +1,14 @@
+"""Stripe analytics source settings and constants"""
+
+# the most popular endpoints
+# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
+ENDPOINTS = (
+    "Subscription",
+    "Account",
+    "Coupon",
+    "Customer",
+    "Product",
+    "Price",
+)
+# possible incremental endpoints
+INCREMENTAL_ENDPOINTS = ("Event", "Invoice", "BalanceTransaction")

ingestr-0.7.5/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.7.5"

{ingestr-0.7.3 → ingestr-0.7.5}/pyproject.toml

@@ -68,6 +68,7 @@ exclude = [
     'src/google_sheets/.*',
     'src/shopify/.*',
     'src/gorgias/.*',
+    'src/stripe_analytics/.*'
 ]
 
 [[tool.mypy.overrides]]
@@ -77,6 +78,7 @@ module = [
   "ingestr.src.google_sheets.*",
   "ingestr.src.shopify.*",
   "ingestr.src.gorgias.*",
+  "ingestr.src.stripe_analytics.*",
 ]
 follow_imports = "skip"
 

{ingestr-0.7.3 → ingestr-0.7.5}/requirements.txt

@@ -1,4 +1,3 @@
-connectorx==0.3.3
 cx_Oracle==8.3.0
 databricks-sql-connector==2.9.3
 dlt==0.5.1
@@ -25,3 +24,4 @@ SQLAlchemy==1.4.52
 sqlalchemy2-stubs==0.0.2a38
 tqdm==4.66.2
 typer==0.12.3
+stripe==10.7.0

ingestr-0.7.3/Dockerfile

@@ -1,45 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-COPY ./requirements.txt /app/requirements.txt
-
-# Setup dependencies for pyodbc
-RUN \
-  export ACCEPT_EULA='Y' && \
-  export MYSQL_CONNECTOR='mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit' && \
-  export MYSQL_CONNECTOR_CHECKSUM='41d03d5df0c631f8071cc697f7714620' && \
-  # Install build dependencies
-  apt-get update && \
-  apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https && \
-  # Install pyodbc db drivers for MSSQL, PG and MySQL
-  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
-  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list && \
-  # install the mysql connector
-  curl -L -o ${MYSQL_CONNECTOR}.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/${MYSQL_CONNECTOR}.tar.gz && \
-  echo "${MYSQL_CONNECTOR_CHECKSUM} ${MYSQL_CONNECTOR}.tar.gz" | md5sum -c - && \
-  gunzip ${MYSQL_CONNECTOR}.tar.gz && tar xvf ${MYSQL_CONNECTOR}.tar && \
-  cp -r ${MYSQL_CONNECTOR}/bin/* /usr/local/bin && cp -r ${MYSQL_CONNECTOR}/lib/* /usr/local/lib && \
-  myodbc-installer -a -d -n "MySQL ODBC 8.0.33 Driver" -t "Driver=/usr/local/lib/libmyodbc8w.so" && \
-  myodbc-installer -a -d -n "MySQL ODBC 8.0.33" -t "Driver=/usr/local/lib/libmyodbc8a.so" && \
-  # install the rest of them
-  apt-get update && \
-  ACCEPT_EULA=Y apt-get install -y msodbcsql17 msodbcsql18 odbc-postgresql && \
-  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
-  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
-  mv -f /tmp/temp.ini /etc/odbcinst.ini && \
-  # Cleanup build dependencies
-  rm -rf ${MYSQL_CONNECTOR}*
-
-
-ENV VIRTUAL_ENV=/usr/local
-ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
-RUN /install.sh && rm /install.sh
-
-RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
-
-COPY . /app
-
-RUN pip3 install -e .
-
-ENTRYPOINT ["ingestr"]

ingestr-0.7.3/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.7.3"