ingestr 0.13.13__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/revenuecat/helpers.py

@@ -0,0 +1,237 @@
+import asyncio
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+import aiohttp
+import pendulum
+import requests
+
+REVENUECAT_API_BASE = "https://api.revenuecat.com/v2"
+
+
+def _make_request(
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make a REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params or {})
+
+            # Handle rate limiting (429 Too Many Requests)
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    # Wait based on Retry-After header or exponential backoff
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                    time.sleep(wait_time)
+                    continue
+
+            response.raise_for_status()
+            return response.json()
+
+        except requests.exceptions.RequestException:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                time.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    response.raise_for_status()
+    return response.json()
+
+
+def _paginate(
+    api_key: str, endpoint: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Paginate through RevenueCat API results."""
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = _make_request(api_key, endpoint, current_params)
+
+        if "items" in data and data["items"] is not None:
+            yield data["items"]
+
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+
+def convert_timestamps_to_iso(
+    record: Dict[str, Any], timestamp_fields: List[str]
+) -> Dict[str, Any]:
+    """Convert timestamp fields from milliseconds to ISO format."""
+    for field in timestamp_fields:
+        if field in record and record[field] is not None:
+            timestamp_ms = record[field]
+            dt = pendulum.from_timestamp(timestamp_ms / 1000)
+            record[field] = dt.to_iso8601_string()
+
+    return record
+
+
+async def _make_request_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make an async REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            async with session.get(
+                url, headers=headers, params=params or {}
+            ) as response:
+                # Handle rate limiting (429 Too Many Requests)
+                if response.status == 429:
+                    if attempt < max_retries:
+                        # Wait based on Retry-After header or exponential backoff
+                        retry_after = response.headers.get("Retry-After")
+                        if retry_after:
+                            wait_time = int(retry_after)
+                        else:
+                            wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                response.raise_for_status()
+                return await response.json()
+
+        except aiohttp.ClientError:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                await asyncio.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    async with session.get(url, headers=headers, params=params or {}) as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """Paginate through RevenueCat API results asynchronously."""
+    items = []
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = await _make_request_async(session, api_key, endpoint, current_params)
+
+        # Collect items from the current page
+        if "items" in data and data["items"] is not None:
+            items.extend(data["items"])
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+    return items
+
+
+async def process_customer_with_nested_resources_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer: Dict[str, Any],
+) -> Dict[str, Any]:
+    customer_id = customer["id"]
+    customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
+    nested_resources = [
+        ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
+        ("purchases", ["purchased_at", "expires_at"]),
+    ]
+
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
+
+    return customer
+
+
+def create_project_resource(
+    resource_name: str,
+    api_key: str,
+    project_id: str = None,
+    timestamp_fields: List[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to create DLT resources for project-dependent endpoints.
+
+    Args:
+        resource_name: Name of the resource (e.g., 'products', 'entitlements', 'offerings')
+        api_key: RevenueCat API key
+        project_id: RevenueCat project ID
+        timestamp_fields: List of timestamp fields to convert to ISO format
+
+    Returns:
+        Iterator of resource data
+    """
+    if project_id is None:
+        raise ValueError(f"project_id is required for {resource_name} resource")
+
+    endpoint = f"/projects/{project_id}/{resource_name}"
+    default_timestamp_fields = timestamp_fields or ["created_at", "updated_at"]
+
+    for item in _paginate(api_key, endpoint):
+        item = convert_timestamps_to_iso(item, default_timestamp_fields)
+        yield item

ingestr/src/salesforce/__init__.py

@@ -13,6 +13,8 @@ def salesforce_source(
     username: str,
     password: str,
     token: str,
+    domain: str,
+    custom_object: str = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from Salesforce using the Salesforce API.
@@ -26,7 +28,7 @@ def salesforce_source(
         DltResource: Data resources from Salesforce.
     """
 
-    client = Salesforce(username, password, token)
+    client = Salesforce(username, password, token, domain=domain)
 
     # define resources
     @dlt.resource(write_disposition="replace")
@@ -37,7 +39,7 @@ def salesforce_source(
     def user_role() -> Iterable[TDataItem]:
         yield get_records(client, "UserRole")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -47,7 +49,7 @@ def salesforce_source(
             client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_line_item(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -57,7 +59,7 @@ def salesforce_source(
             client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_contact_role(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -70,7 +72,7 @@ def salesforce_source(
             "SystemModstamp",
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def account(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "LastModifiedDate", initial_value=None
@@ -92,7 +94,7 @@ def salesforce_source(
     def campaign() -> Iterable[TDataItem]:
         yield get_records(client, "Campaign")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def campaign_member(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -114,7 +116,7 @@ def salesforce_source(
     def pricebook_entry() -> Iterable[TDataItem]:
         yield get_records(client, "PricebookEntry")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def task(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -122,7 +124,7 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def event(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -130,6 +132,10 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
 
+    @dlt.resource(write_disposition="replace")
+    def custom() -> Iterable[TDataItem]:
+        yield get_records(client, custom_object)
+
     return (
         user,
         user_role,
@@ -146,4 +152,5 @@ def salesforce_source(
         pricebook_entry,
         task,
         event,
+        custom,
     )

ingestr/src/shopify/__init__.py

@@ -669,7 +669,7 @@ def shopify_source(
             params["updated_at_max"] = updated_at.end_value.isoformat()
         yield from client.get_pages("customers", params)
 
-    @dlt.resource(primary_key="id", write_disposition="append")
+    @dlt.resource(primary_key="id", write_disposition="merge")
     def events(
         created_at: dlt.sources.incremental[
             pendulum.DateTime
@@ -1690,16 +1690,6 @@ query discountNodes($after: String, $query: String, $first: Int)  {
                 "nullable": True,
                 "description": "The category of the product from Shopify's Standard Product Taxonomy.",
             },
-            "combinedListing": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "A special product type that combines separate products into a single product listing.",
-            },
-            "combinedListingRole": {
-                "data_type": "json",
-                "nullable": True,
-                "description": "The role of the product in a combined listing.",
-            },
             "compareAtPriceRange": {
                 "data_type": "json",
                 "nullable": True,
@@ -1841,12 +1831,6 @@ query products($after: String, $query: String, $first: Int)  {
       category {
         id
       }
-      combinedListing {
-        parentProduct {
-          id
-        }
-      }
-      combinedListingRole
       compareAtPriceRange {
         maxVariantCompareAtPrice {
           amount

ingestr/src/smartsheets/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import smartsheet  # type: ignore
+from dlt.extract import DltResource
+from smartsheet.models.enums import ColumnType  # type: ignore
+from smartsheet.models.sheet import Sheet  # type: ignore
+
+TYPE_MAPPING = {
+    ColumnType.TEXT_NUMBER: "text",
+    ColumnType.DATE: "date",
+    ColumnType.DATETIME: "timestamp",
+    ColumnType.CONTACT_LIST: "text",
+    ColumnType.CHECKBOX: "bool",
+    ColumnType.PICKLIST: "text",
+    ColumnType.DURATION: "text",
+    ColumnType.PREDECESSOR: "text",
+    ColumnType.ABSTRACT_DATETIME: "timestamp",
+    ColumnType.MULTI_CONTACT_LIST: "text",
+    ColumnType.MULTI_PICKLIST: "text",
+}
+
+
+@dlt.source
+def smartsheet_source(
+    access_token: str,
+    sheet_id: str,
+) -> Iterable[DltResource]:
+    """
+    A DLT source for Smartsheet.
+
+    Args:
+        access_token: The Smartsheet API access token.
+        sheet_id: The ID of the sheet to load.
+
+    Returns:
+        An iterable of DLT resources.
+    """
+
+    # Initialize Smartsheet client
+    smartsheet_client = smartsheet.Smartsheet(access_token)
+    smartsheet_client.errors_as_exceptions(True)
+
+    # The SDK expects sheet_id to be an int
+    sheet_id_int = int(sheet_id)
+    # Sanitize the sheet name to be a valid resource name
+    # We get objectValue to ensure `name` attribute is populated for the sheet
+    sheet_details = smartsheet_client.Sheets.get_sheet(
+        sheet_id_int, include=["objectValue"]
+    )
+    sheet_name = sheet_details.name
+    resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id_int)
+
+    yield dlt.resource(
+        _get_sheet_data(sheet),
+        name=resource_name,
+        columns=_generate_type_hints(sheet),
+        write_disposition="replace",
+    )
+
+
+def _get_sheet_data(sheet: Sheet):
+    """Helper function to get all rows from a sheet."""
+
+    column_titles = [col.title for col in sheet.columns]
+    for row in sheet.rows:
+        row_data = {"_row_id": row.id}
+        for i, cell in enumerate(row.cells):
+            row_data[column_titles[i]] = cell.value
+        yield row_data
+
+
+def _generate_type_hints(sheet: Sheet):
+    return {
+        col.title: {
+            "data_type": TYPE_MAPPING.get(col.type.value),
+            "nullable": True,
+        }
+        for col in sheet.columns
+        if col.type.value in TYPE_MAPPING
+    }