ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/revenuecat/helpers.py

@@ -0,0 +1,237 @@
+import asyncio
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+import aiohttp
+import pendulum
+import requests
+
+REVENUECAT_API_BASE = "https://api.revenuecat.com/v2"
+
+
+def _make_request(
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make a REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params or {})
+
+            # Handle rate limiting (429 Too Many Requests)
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    # Wait based on Retry-After header or exponential backoff
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                    time.sleep(wait_time)
+                    continue
+
+            response.raise_for_status()
+            return response.json()
+
+        except requests.exceptions.RequestException:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                time.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    response.raise_for_status()
+    return response.json()
+
+
+def _paginate(
+    api_key: str, endpoint: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Paginate through RevenueCat API results."""
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = _make_request(api_key, endpoint, current_params)
+
+        if "items" in data and data["items"] is not None:
+            yield data["items"]
+
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+
+def convert_timestamps_to_iso(
+    record: Dict[str, Any], timestamp_fields: List[str]
+) -> Dict[str, Any]:
+    """Convert timestamp fields from milliseconds to ISO format."""
+    for field in timestamp_fields:
+        if field in record and record[field] is not None:
+            timestamp_ms = record[field]
+            dt = pendulum.from_timestamp(timestamp_ms / 1000)
+            record[field] = dt.to_iso8601_string()
+
+    return record
+
+
+async def _make_request_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make an async REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            async with session.get(
+                url, headers=headers, params=params or {}
+            ) as response:
+                # Handle rate limiting (429 Too Many Requests)
+                if response.status == 429:
+                    if attempt < max_retries:
+                        # Wait based on Retry-After header or exponential backoff
+                        retry_after = response.headers.get("Retry-After")
+                        if retry_after:
+                            wait_time = int(retry_after)
+                        else:
+                            wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                response.raise_for_status()
+                return await response.json()
+
+        except aiohttp.ClientError:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                await asyncio.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    async with session.get(url, headers=headers, params=params or {}) as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """Paginate through RevenueCat API results asynchronously."""
+    items = []
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = await _make_request_async(session, api_key, endpoint, current_params)
+
+        # Collect items from the current page
+        if "items" in data and data["items"] is not None:
+            items.extend(data["items"])
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+    return items
+
+
+async def process_customer_with_nested_resources_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer: Dict[str, Any],
+) -> Dict[str, Any]:
+    customer_id = customer["id"]
+    customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
+    nested_resources = [
+        ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
+        ("purchases", ["purchased_at", "expires_at"]),
+    ]
+
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
+
+    return customer
+
+
+def create_project_resource(
+    resource_name: str,
+    api_key: str,
+    project_id: str = None,
+    timestamp_fields: List[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to create DLT resources for project-dependent endpoints.
+
+    Args:
+        resource_name: Name of the resource (e.g., 'products', 'entitlements', 'offerings')
+        api_key: RevenueCat API key
+        project_id: RevenueCat project ID
+        timestamp_fields: List of timestamp fields to convert to ISO format
+
+    Returns:
+        Iterator of resource data
+    """
+    if project_id is None:
+        raise ValueError(f"project_id is required for {resource_name} resource")
+
+    endpoint = f"/projects/{project_id}/{resource_name}"
+    default_timestamp_fields = timestamp_fields or ["created_at", "updated_at"]
+
+    for item in _paginate(api_key, endpoint):
+        item = convert_timestamps_to_iso(item, default_timestamp_fields)
+        yield item

ingestr/src/salesforce/__init__.py

@@ -0,0 +1,156 @@
+from typing import Iterable
+
+import dlt
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource, incremental
+from simple_salesforce import Salesforce
+
+from .helpers import get_records
+
+
+@dlt.source(name="salesforce")
+def salesforce_source(
+    username: str,
+    password: str,
+    token: str,
+    domain: str,
+    custom_object: str = None,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from Salesforce using the Salesforce API.
+
+    Args:
+        username (str): The username for authentication.
+        password (str): The password for authentication.
+        token (str): The security token for authentication.
+
+    Yields:
+        DltResource: Data resources from Salesforce.
+    """
+
+    client = Salesforce(username, password, token, domain=domain)
+
+    # define resources
+    @dlt.resource(write_disposition="replace")
+    def user() -> Iterable[TDataItem]:
+        yield get_records(client, "User")
+
+    @dlt.resource(write_disposition="replace")
+    def user_role() -> Iterable[TDataItem]:
+        yield get_records(client, "UserRole")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity_line_item(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def opportunity_contact_role(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client,
+            "OpportunityContactRole",
+            last_timestamp.last_value,
+            "SystemModstamp",
+        )
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def account(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "LastModifiedDate", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "Account", last_timestamp.last_value, "LastModifiedDate"
+        )
+
+    @dlt.resource(write_disposition="replace")
+    def contact() -> Iterable[TDataItem]:
+        yield get_records(client, "Contact")
+
+    @dlt.resource(write_disposition="replace")
+    def lead() -> Iterable[TDataItem]:
+        yield get_records(client, "Lead")
+
+    @dlt.resource(write_disposition="replace")
+    def campaign() -> Iterable[TDataItem]:
+        yield get_records(client, "Campaign")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def campaign_member(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(
+            client, "CampaignMember", last_timestamp.last_value, "SystemModstamp"
+        )
+
+    @dlt.resource(write_disposition="replace")
+    def product() -> Iterable[TDataItem]:
+        yield get_records(client, "Product2")
+
+    @dlt.resource(write_disposition="replace")
+    def pricebook() -> Iterable[TDataItem]:
+        yield get_records(client, "Pricebook2")
+
+    @dlt.resource(write_disposition="replace")
+    def pricebook_entry() -> Iterable[TDataItem]:
+        yield get_records(client, "PricebookEntry")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def task(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def event(
+        last_timestamp: incremental[str] = dlt.sources.incremental(
+            "SystemModstamp", initial_value=None
+        ),
+    ) -> Iterable[TDataItem]:
+        yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
+
+    @dlt.resource(write_disposition="replace")
+    def custom() -> Iterable[TDataItem]:
+        yield get_records(client, custom_object)
+
+    return (
+        user,
+        user_role,
+        opportunity,
+        opportunity_line_item,
+        opportunity_contact_role,
+        account,
+        contact,
+        lead,
+        campaign,
+        campaign_member,
+        product,
+        pricebook,
+        pricebook_entry,
+        task,
+        event,
+        custom,
+    )

ingestr/src/salesforce/helpers.py

@@ -0,0 +1,64 @@
+"""Salesforce source helpers"""
+
+from typing import Iterable, Optional
+
+import pendulum
+from dlt.common.typing import TDataItem
+from simple_salesforce import Salesforce
+
+
+def get_records(
+    sf: Salesforce,
+    sobject: str,
+    last_state: Optional[str] = None,
+    replication_key: Optional[str] = None,
+) -> Iterable[TDataItem]:
+    """
+    Retrieves records from Salesforce for a specified sObject.
+
+    Args:
+        sf (Salesforce): An instance of the Salesforce API client.
+        sobject (str): The name of the sObject to retrieve records from.
+        last_state (str, optional): The last known state for incremental loading. Defaults to None.
+        replication_key (str, optional): The replication key for incremental loading. Defaults to None.
+
+    Yields:
+        Dict[TDataItem]: A dictionary representing a record from the Salesforce sObject.
+    """
+
+    # Get all fields for the sobject
+    desc = getattr(sf, sobject).describe()
+    # Salesforce returns compound fields as separate fields, so we need to filter them out
+    compound_fields = {
+        f["compoundFieldName"]
+        for f in desc["fields"]
+        if f["compoundFieldName"] is not None
+    } - {"Name"}
+    # Salesforce returns datetime fields as timestamps, so we need to convert them
+    date_fields = {
+        f["name"] for f in desc["fields"] if f["type"] in ("datetime",) and f["name"]
+    }
+    # If no fields are specified, use all fields except compound fields
+    fields = [f["name"] for f in desc["fields"] if f["name"] not in compound_fields]
+
+    # Generate a predicate to filter records by the replication key
+    predicate, order_by, n_records = "", "", 0
+    if replication_key:
+        if last_state:
+            predicate = f"WHERE {replication_key} > {last_state}"
+        order_by = f"ORDER BY {replication_key} ASC"
+    query = f"SELECT {', '.join(fields)} FROM {sobject} {predicate} {order_by}"
+
+    # Query all records in batches
+    for page in getattr(sf.bulk, sobject).query_all(query, lazy_operation=True):
+        for record in page:
+            # Strip out the attributes field
+            record.pop("attributes", None)
+            for field in date_fields:
+                # Convert Salesforce timestamps to ISO 8601
+                if record.get(field):
+                    record[field] = pendulum.from_timestamp(
+                        record[field] / 1000,
+                    ).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+        yield from page
+        n_records += len(page)

ingestr/src/shopify/__init__.py

@@ -669,7 +669,7 @@ def shopify_source(
             params["updated_at_max"] = updated_at.end_value.isoformat()
         yield from client.get_pages("customers", params)
 
-    @dlt.resource(primary_key="id", write_disposition="append")
+    @dlt.resource(primary_key="id", write_disposition="merge")
     def events(
         created_at: dlt.sources.incremental[
             pendulum.DateTime
@@ -1690,16 +1690,6 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "nullable": True,
                 "description": "The category of the product from Shopify's Standard Product Taxonomy.",
             },
-            "combinedListing": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "A special product type that combines separate products into a single product listing.",
-            },
-            "combinedListingRole": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "The role of the product in a combined listing.",
-            },
             "compareAtPriceRange": {
                 "data_type": "json",
                 "nullable": True,
@@ -1841,12 +1831,6 @@ query products($after: String, $query: String, $first: Int)  {
       category {
         id
       }
-      combinedListing {
-        parentProduct {
-          id
-        }
-      }
-      combinedListingRole
       compareAtPriceRange {
         maxVariantCompareAtPrice {
           amount

ingestr/src/smartsheets/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import smartsheet  # type: ignore
+from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
+
+
+@dlt.source
+def smartsheet_source(
+    access_token: str,
+    sheet_id: str,
+) -> Iterable[DltResource]:
+    """
+    A DLT source for Smartsheet.
+
+    Args:
+        access_token: The Smartsheet API access token.
+        sheet_id: The ID of the sheet to load.
+
+    Returns:
+        An iterable of DLT resources.
+    """
+
+    # Initialize Smartsheet client
+    smartsheet_client = smartsheet.Smartsheet(access_token)
+    smartsheet_client.errors_as_exceptions(True)
+
+    # The SDK expects sheet_id to be an int
+    sheet_id_int = int(sheet_id)
+    # Sanitize the sheet name to be a valid resource name
+    # We get objectValue to ensure `name` attribute is populated for the sheet
+    sheet_details = smartsheet_client.Sheets.get_sheet(
+        sheet_id_int, include=["objectValue"]
+    )
+    sheet_name = sheet_details.name
+    resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
+
+    yield dlt.resource(
+        _get_sheet_data(sheet),
+        name=resource_name,
+        columns=_generate_type_hints(sheet),
+        write_disposition="replace",
+    )
+
+
+def _get_sheet_data(sheet: Sheet):
+    """Helper function to get all rows from a sheet."""
+
+    column_titles = [col.title for col in sheet.columns]
+    for row in sheet.rows:
+        row_data = {"_row_id": row.id}
+        for i, cell in enumerate(row.cells):
+            row_data[column_titles[i]] = cell.value
+        yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }