ingestr 0.13.53__tar.gz → 0.13.54__tar.gz

{ingestr-0.13.53 → ingestr-0.13.54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.53
+Version: 0.13.54
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -52,14 +52,17 @@ Requires-Dist: dlt==1.10.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1
+Requires-Dist: ecdsa==0.19.1
 Requires-Dist: elastic-transport==8.17.1
 Requires-Dist: elasticsearch==8.10.1
+Requires-Dist: enum-compat==0.0.3
 Requires-Dist: et-xmlfile==2.0.0
 Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
+Requires-Dist: future==1.0.0
 Requires-Dist: gcsfs==2025.3.2
 Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
@@ -93,6 +96,7 @@ Requires-Dist: ibm-db-sa==0.4.1
 Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
+Requires-Dist: intuit-oauth==1.2.4
 Requires-Dist: isodate==0.7.2
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: jsonpath-ng==1.7.0
@@ -147,8 +151,11 @@ Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: python-dateutil==2.9.0.post0
 Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-jose==3.5.0
+Requires-Dist: python-quickbooks==0.9.2
 Requires-Dist: pytz==2025.1
 Requires-Dist: pyyaml==6.0.2
+Requires-Dist: rauth==0.7.3
 Requires-Dist: redshift-connector==2.1.5
 Requires-Dist: requests-file==2.1.0
 Requires-Dist: requests-oauthlib==1.3.1

{ingestr-0.13.53 → ingestr-0.13.54}/docs/.vitepress/config.mjs

@@ -132,11 +132,13 @@ export default defineConfig({
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
+              { text: "Mixpanel", link: "/supported-sources/mixpanel.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "Personio", link: "/supported-sources/personio.md" },
               { text: "PhantomBuster", link: "/supported-sources/phantombuster.md" },
               { text: "Pipedrive", link: "/supported-sources/pipedrive.md" },
-              { text: "S3", link: "/supported-sources/s3.md"},
+              { text: "QuickBooks", link: "/supported-sources/quickbooks.md" },
+              { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "SFTP", link: "/supported-sources/sftp.md"},
               { text: "Shopify", link: "/supported-sources/shopify.md" },

ingestr-0.13.54/docs/supported-sources/mixpanel.md

@@ -0,0 +1,46 @@
+# Mixpanel
+
+[Mixpanel](https://mixpanel.com/) is an analytics service for tracking user interactions in web and mobile applications.
+
+ingestr supports Mixpanel as a source.
+
+## URI format
+
+```plaintext
+mixpanel://?username=<service_account_username>&password=<service_account_secret>&project_id=<project_id>&server=<server>
+```
+
+URI parameters:
+
+- `username`: Mixpanel service account username.
+- `password`: Mixpanel service account secret. This is the secret associated with the service account.
+- `project_id`: The numeric project ID.
+- `server`: (Optional) The server region to use. Can be "us", "eu", or "in". Defaults to "eu".
+
+
+To grab mixpanel credentials, please follow the guide [here](https://developer.mixpanel.com/reference/service-accounts).
+
+## Example
+
+Copy events from Mixpanel into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'mixpanel://?username=my-service-account&password=my-secret&project_id=12345' \
+    --source-table 'events' \
+    --dest-uri duckdb:///mixpanel.duckdb \
+    --dest-table 'mixpanel.events'
+```
+
+
+<img alt="mixpanel" src="../media/mixpanel_ingestion.png"/>
+
+
+## Tables
+
+Mixpanel source allows ingesting the following tables:
+
+- `events`: Raw event data returned from the export API.
+- `profiles`: User profiles from the Engage API.
+
+Use these as `--source-table` values in the `ingestr ingest` command.

ingestr-0.13.54/docs/supported-sources/quickbooks.md

@@ -0,0 +1,49 @@
+# QuickBooks
+
+[QuickBooks](https://quickbooks.intuit.com/) is an accounting software package developed and marketed by Intuit.
+
+ingestr supports QuickBooks as a source.
+
+## URI format
+
+```plaintext
+quickbooks://?company_id=<company_id>client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh_token>&access_token=<access_token>&environment=<environment>&minor_version=<minor_version>
+```
+
+URI parameters:
+- `company_id`: The QuickBooks company (realm) id.
+- `client_id`: OAuth client id from your Intuit application.
+- `client_secret`: OAuth client secret.
+- `refresh_token`: OAuth refresh token used to obtain access tokens.
+- `environment`: Optional environment name, either `production` or `sandbox`. Defaults to `production`.
+- `minor_version`: Optional API minor version.
+
+## Setting up a QuickBooks integration
+
+Follow Intuit's [OAuth setup guide](https://developer.intuit.com/app/developer/qbo/docs/develop/authentication-and-authorization) to create an app and generate your credentials.
+
+Once you have the credentials, you can ingest data. For example, to copy customers data into DuckDB:
+
+```sh
+ingestr ingest \
+--source-uri 'quickbooks://?company_id=1234567890&client_id=cid&client_secret=csecret&refresh_token=rtoken' \
+--source-table 'customers' \
+--dest-uri duckdb:///customers.duckdb \
+--dest-table 'dest.details'
+```
+This command will retrieve customers data and save it to the `dest.details` table in the DuckDB database.
+
+<img alt="quickbooks" src="../media/quickbook_ingestion.png"/>
+
+
+## Tables
+QuickBooks source allows ingesting the following tables:
+
+- `customers`: Retrives list of customers.
+- `invoices`: Retrives sales invoices.
+- `accounts`: Retrives details of accounts.
+- `vendors`: Retrives vendor records.
+- `payments`: Retrives payments recorded.
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.
+

ingestr-0.13.54/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.54"

{ingestr-0.13.53 → ingestr-0.13.54}/ingestr/src/destinations.py

@@ -476,7 +476,7 @@ class SqliteDestination(GenericSqlDestination):
 
     def dlt_run_params(self, uri: str, table: str, **kwargs):
         return {
-            #https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            # https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
             "dataset_name": "main",
             "table_name": table,
         }
@@ -495,6 +495,3 @@ class MySqlDestination(GenericSqlDestination):
             "dataset_name": database,
             "table_name": table,
         }
-
-
-

{ingestr-0.13.53 → ingestr-0.13.54}/ingestr/src/factory.py

@@ -47,11 +47,13 @@ from ingestr.src.sources import (
     KlaviyoSource,
     LinkedInAdsSource,
     LocalCsvSource,
+    MixpanelSource,
     MongoDbSource,
     NotionSource,
     PersonioSource,
     PhantombusterSource,
     PipedriveSource,
+    QuickBooksSource,
     S3Source,
     SalesforceSource,
     SFTPSource,
@@ -140,6 +142,7 @@ class SourceDestinationFactory:
         "hubspot": HubspotSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
+        "mixpanel": MixpanelSource,
         "appsflyer": AppsflyerSource,
         "kafka": KafkaSource,
         "adjust": AdjustSource,
@@ -166,6 +169,7 @@ class SourceDestinationFactory:
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,
         "solidgate": SolidgateSource,
+        "quickbooks": QuickBooksSource,
         "smartsheet": SmartsheetSource,
         "sftp": SFTPSource,
     }

ingestr-0.13.54/ingestr/src/mixpanel/__init__.py

@@ -0,0 +1,62 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import MixpanelClient
+
+
+@dlt.source(max_table_nesting=0)
+def mixpanel_source(
+    username: str,
+    password: str,
+    project_id: str,
+    server: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = MixpanelClient(username, password, project_id, server)
+
+    @dlt.resource(write_disposition="merge", name="events", primary_key="distinct_id")
+    def events(
+        date=dlt.sources.incremental(
+            "time",
+            initial_value=start_date.int_timestamp,
+            end_value=end_date.int_timestamp if end_date else None,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if date.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = pendulum.from_timestamp(date.end_value)
+
+        start_dt = pendulum.from_timestamp(date.last_value)
+
+        yield from client.fetch_events(
+            start_dt,
+            end_dt,
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="distinct_id", name="profiles")
+    def profiles(
+        last_seen=dlt.sources.incremental(
+            "last_seen",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if last_seen.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = last_seen.end_value
+
+        start_dt = last_seen.last_value
+        yield from client.fetch_profiles(start_dt, end_dt)
+
+    return events, profiles

ingestr-0.13.54/ingestr/src/mixpanel/client.py

@@ -0,0 +1,99 @@
+import json
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class MixpanelClient:
+    def __init__(self, username: str, password: str, project_id: str, server: str):
+        self.username = username
+        self.password = password
+        self.project_id = project_id
+        self.server = server
+        self.session = Client(raise_for_status=False).session
+
+    def fetch_events(
+        self, start_date: pendulum.DateTime, end_date: pendulum.DateTime
+    ) -> Iterable[dict]:
+        if self.server == "us":
+            server = "data"
+        elif self.server == "in":
+            server = "data-in"
+        else:
+            server = "data-eu"
+
+        url = f"https://{server}.mixpanel.com/api/2.0/export/"
+        params = {
+            "project_id": self.project_id,
+            "from_date": start_date.format("YYYY-MM-DD"),
+            "to_date": end_date.format("YYYY-MM-DD"),
+        }
+        headers = {
+            "accept": "text/plain",
+        }
+        from requests.auth import HTTPBasicAuth
+
+        auth = HTTPBasicAuth(self.username, self.password)
+        resp = self.session.get(url, params=params, headers=headers, auth=auth)
+        resp.raise_for_status()
+        for line in resp.iter_lines():
+            if line:
+                data = json.loads(line.decode())
+                if "properties" in data:
+                    for key, value in data["properties"].items():
+                        if key.startswith("$"):
+                            data[key[1:]] = value
+                        else:
+                            data[key] = value
+                    del data["properties"]
+                yield data
+
+    def fetch_profiles(
+        self, start_date: pendulum.DateTime, end_date: pendulum.DateTime
+    ) -> Iterable[dict]:
+        if self.server == "us":
+            server = ""
+        elif self.server == "in":
+            server = "in."
+        else:
+            server = "eu."
+        url = f"https://{server}mixpanel.com/api/query/engage"
+        headers = {
+            "accept": "application/json",
+            "content-type": "application/x-www-form-urlencoded",
+        }
+        from requests.auth import HTTPBasicAuth
+
+        auth = HTTPBasicAuth(self.username, self.password)
+        page = 0
+        session_id = None
+        while True:
+            params = {"project_id": self.project_id, "page": str(page)}
+            if session_id:
+                params["session_id"] = session_id
+            start_str = start_date.format("YYYY-MM-DDTHH:mm:ss")
+            end_str = end_date.format("YYYY-MM-DDTHH:mm:ss")
+            where = f'properties["$last_seen"] >= "{start_str}" and properties["$last_seen"] <= "{end_str}"'
+            params["where"] = where
+            resp = self.session.post(url, params=params, headers=headers, auth=auth)
+
+            resp.raise_for_status()
+            data = resp.json()
+
+            for result in data.get("results", []):
+                for key, value in result["$properties"].items():
+                    if key.startswith("$"):
+                        if key == "$last_seen":
+                            result["last_seen"] = pendulum.parse(value)
+                        else:
+                            result[key[1:]] = value
+                result["distinct_id"] = result["$distinct_id"]
+                del result["$properties"]
+                del result["$distinct_id"]
+                yield result
+            if not data.get("results"):
+                break
+            session_id = data.get("session_id", session_id)
+
+            page += 1

ingestr-0.13.54/ingestr/src/quickbooks/__init__.py

@@ -0,0 +1,117 @@
+"""QuickBooks source built on top of python-quickbooks."""
+
+from typing import Iterable, Iterator, List, Optional
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from intuitlib.client import AuthClient  # type: ignore
+
+from quickbooks import QuickBooks  # type: ignore
+
+
+@dlt.source(name="quickbooks", max_table_nesting=0)
+def quickbooks_source(
+    company_id: str,
+    start_date: pendulum.DateTime,
+    object: str,
+    end_date: pendulum.DateTime | None,
+    client_id: str,
+    client_secret: str,
+    refresh_token: str,
+    environment: str = "production",
+    minor_version: Optional[str] = None,
+) -> Iterable[DltResource]:
+    """Create dlt resources for QuickBooks objects.
+
+    Parameters
+    ----------
+    company_id: str
+        QuickBooks company id (realm id).
+    client_id: str
+        OAuth client id.
+    client_secret: str
+        OAuth client secret.
+    refresh_token: str
+        OAuth refresh token.
+    access_token: Optional[str]
+        Optional access token. If not provided the library will refresh using the
+        provided refresh token.
+    environment: str
+        Either ``"production"`` or ``"sandbox"``.
+    minor_version: Optional[int]
+        QuickBooks API minor version if needed.
+    """
+
+    auth_client = AuthClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        environment=environment,
+        # redirect_uri is not used since we authenticate using refresh token which skips the step of redirect callback.
+        # as redirect_uri is required param, we are passing empty string.
+        redirect_uri="",
+    )
+
+    # https://help.developer.intuit.com/s/article/Validity-of-Refresh-Token
+    client = QuickBooks(
+        auth_client=auth_client,
+        refresh_token=refresh_token,
+        company_id=company_id,
+        minorversion=minor_version,
+    )
+
+    def fetch_object(
+        obj_name: str,
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "lastupdatedtime",
+            initial_value=start_date,  # type: ignore
+            end_value=end_date,  # type: ignore
+            range_start="closed",
+            range_end="closed",
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[List[TDataItem]]:
+        start_pos = 1
+
+        end_dt = updated_at.end_value or pendulum.now(tz="UTC")
+        start_dt = ensure_pendulum_datetime(str(updated_at.last_value)).in_tz("UTC")
+
+        start_str = start_dt.isoformat()
+        end_str = end_dt.isoformat()
+
+        where_clause = f"WHERE MetaData.LastUpdatedTime >= '{start_str}' AND MetaData.LastUpdatedTime < '{end_str}'"
+        while True:
+            query = (
+                f"SELECT * FROM {obj_name} {where_clause} "
+                f"ORDERBY MetaData.LastUpdatedTime ASC STARTPOSITION {start_pos} MAXRESULTS 1000"
+            )
+
+            result = client.query(query)
+
+            items = result.get("QueryResponse", {}).get(obj_name.capitalize(), [])
+            if not items:
+                break
+
+            for item in items:
+                if item.get("MetaData") and item["MetaData"].get("LastUpdatedTime"):
+                    item["lastupdatedtime"] = ensure_pendulum_datetime(
+                        item["MetaData"]["LastUpdatedTime"]
+                    )
+                    item["id"] = item["Id"]
+                    del item["Id"]
+
+                yield item
+
+            if len(items) < 1000:
+                break
+
+            start_pos += 1000
+
+    yield dlt.resource(
+        fetch_object,
+        name=object.lower(),
+        write_disposition="merge",
+        primary_key="id",
+    )(object)

{ingestr-0.13.53 → ingestr-0.13.54}/ingestr/src/sources.py

@@ -79,7 +79,7 @@ class SqlSource:
         # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
         if uri.startswith("clickhouse://"):
             parsed_uri = urlparse(uri)
-            
+
             query_params = parse_qs(parsed_uri.query)
 
             if "http_port" in query_params:
@@ -691,8 +691,6 @@ class StripeAnalyticsSource:
                     endpoint,
                 ],
                 stripe_secret_key=api_key[0],
-                start_date=kwargs.get("interval_start", None),
-                end_date=kwargs.get("interval_end", None),
             ).with_resources(endpoint)
 
         elif table in INCREMENTAL_ENDPOINTS:
@@ -965,6 +963,57 @@ class KlaviyoSource:
         ).with_resources(resource)
 
 
+class MixpanelSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Mixpanel takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        parsed = urlparse(uri)
+        params = parse_qs(parsed.query)
+        username = params.get("username")
+        password = params.get("password")
+        project_id = params.get("project_id")
+        server = params.get("server", ["eu"])
+
+        if not username or not password or not project_id:
+            raise ValueError(
+                "username, password, project_id are required to connect to Mixpanel"
+            )
+
+        if table not in ["events", "profiles"]:
+            raise ValueError(
+                f"Resource '{table}' is not supported for Mixpanel source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+            )
+
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date).in_timezone("UTC")
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_timezone("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date).in_timezone("UTC")
+        else:
+            end_date = pendulum.now().in_timezone("UTC")
+
+        from ingestr.src.mixpanel import mixpanel_source
+
+        return mixpanel_source(
+            username=username[0],
+            password=password[0],
+            project_id=project_id[0],
+            start_date=start_date,
+            end_date=end_date,
+            server=server[0],
+        ).with_resources(table)
+
+
 class KafkaSource:
     def handles_incrementality(self) -> bool:
         return False
@@ -2536,3 +2585,73 @@ class SFTPSource:
 
         dlt_source_resource = readers(bucket_url, fs, file_glob)
         return dlt_source_resource.with_resources(endpoint)
+
+
+class QuickBooksSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    # quickbooks://?company_id=<company_id>&client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh>&access_token=<access_token>&environment=<env>&minor_version=<version>
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+
+        params = parse_qs(parsed_uri.query)
+        company_id = params.get("company_id")
+        client_id = params.get("client_id")
+        client_secret = params.get("client_secret")
+        refresh_token = params.get("refresh_token")
+        environment = params.get("environment", ["production"])
+        minor_version = params.get("minor_version", [None])
+
+        if not client_id or not client_id[0].strip():
+            raise MissingValueError("client_id", "QuickBooks")
+
+        if not client_secret or not client_secret[0].strip():
+            raise MissingValueError("client_secret", "QuickBooks")
+
+        if not refresh_token or not refresh_token[0].strip():
+            raise MissingValueError("refresh_token", "QuickBooks")
+
+        if not company_id or not company_id[0].strip():
+            raise MissingValueError("company_id", "QuickBooks")
+
+        if environment[0] not in ["production", "sandbox"]:
+            raise ValueError(
+                "Invalid environment. Must be either 'production' or 'sandbox'."
+            )
+
+        from ingestr.src.quickbooks import quickbooks_source
+
+        table_name = table.replace(" ", "")
+        table_mapping = {
+            "customers": "customer",
+            "invoices": "invoice",
+            "accounts": "account",
+            "vendors": "vendor",
+            "payments": "payment",
+        }
+        if table_name in table_mapping:
+            table_name = table_mapping[table_name]
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = ensure_pendulum_datetime("2025-01-01").in_tz("UTC")
+        else:
+            start_date = ensure_pendulum_datetime(start_date).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        return quickbooks_source(
+            company_id=company_id[0],
+            start_date=start_date,
+            end_date=end_date,
+            client_id=client_id[0],
+            client_secret=client_secret[0],
+            refresh_token=refresh_token[0],
+            environment=environment[0],
+            minor_version=minor_version[0],
+            object=table_name,
+        ).with_resources(table_name)

{ingestr-0.13.53 → ingestr-0.13.54}/requirements.in

@@ -55,3 +55,4 @@ sqlalchemy-spanner==1.11.0
 google-cloud-spanner==3.54.0
 smartsheet-python-sdk==3.0.5
 paramiko==3.5.1
+python-quickbooks==0.9.2