ingestr 0.13.85__py3-none-any.whl → 0.13.87__py3-none-any.whl

ingestr/src/mongodb/__init__.py

@@ -101,7 +101,7 @@ def mongodb_collection(
     write_disposition: Optional[str] = dlt.config.value,
     parallel: Optional[bool] = False,
     limit: Optional[int] = None,
-    chunk_size: Optional[int] = 10000,
+    chunk_size: Optional[int] = 1000,
     data_item_format: Optional[TDataItemFormat] = "object",
     filter_: Optional[Dict[str, Any]] = None,
     projection: Optional[Union[Mapping[str, Any], Iterable[str]]] = dlt.config.value,

ingestr/src/mongodb/helpers.py

@@ -518,21 +518,42 @@ class CollectionAggregationLoader(CollectionLoader):
         if limit and limit > 0:
             pipeline.append({"$limit": limit})
 
-        print("pipeline", pipeline)
-        # Execute aggregation
-        cursor = self.collection.aggregate(pipeline, allowDiskUse=True)
-
-        # Process results in chunks
-        while docs_slice := list(islice(cursor, self.chunk_size)):
-            res = map_nested_in_place(convert_mongo_objs, docs_slice)
-            print("res", res)
-            if len(res) > 0 and "_id" in res[0] and isinstance(res[0]["_id"], dict):
-                yield dlt.mark.with_hints(
-                    res,
-                    dlt.mark.make_hints(columns={"_id": {"data_type": "json"}}),
-                )
-            else:
-                yield res
+        # Add maxTimeMS to prevent hanging
+        cursor = self.collection.aggregate(
+            pipeline, 
+            allowDiskUse=True, 
+            batchSize=min(self.chunk_size, 101),
+            maxTimeMS=30000  # 30 second timeout
+        )
+        
+        docs_buffer = []
+        try:
+            for doc in cursor:
+                docs_buffer.append(doc)
+                
+                if len(docs_buffer) >= self.chunk_size:
+                    res = map_nested_in_place(convert_mongo_objs, docs_buffer)
+                    if len(res) > 0 and "_id" in res[0] and isinstance(res[0]["_id"], dict):
+                        yield dlt.mark.with_hints(
+                            res,
+                            dlt.mark.make_hints(columns={"_id": {"data_type": "json"}}),
+                        )
+                    else:
+                        yield res
+                    docs_buffer = []
+            
+            # Yield any remaining documents
+            if docs_buffer:
+                res = map_nested_in_place(convert_mongo_objs, docs_buffer)
+                if len(res) > 0 and "_id" in res[0] and isinstance(res[0]["_id"], dict):
+                    yield dlt.mark.with_hints(
+                        res,
+                        dlt.mark.make_hints(columns={"_id": {"data_type": "json"}}),
+                    )
+                else:
+                    yield res
+        finally:
+            cursor.close()
 
 
 class CollectionAggregationLoaderParallel(CollectionAggregationLoader):

ingestr/src/revenuecat/__init__.py

@@ -0,0 +1,108 @@
+import asyncio
+from typing import Any, Dict, Iterable, Iterator
+
+import aiohttp
+import dlt
+
+from .helpers import (
+    _make_request,
+    _paginate,
+    convert_timestamps_to_iso,
+    process_customer_with_nested_resources_async,
+    create_project_resource,
+)
+
+
+
+@dlt.source(name="revenuecat", max_table_nesting=0)
+def revenuecat_source(
+    api_key: str,
+    project_id: str = None,
+) -> Iterable[dlt.sources.DltResource]:
+    """
+    RevenueCat source for extracting data from RevenueCat API v2.
+
+    Args:
+        api_key: RevenueCat API v2 secret key with Bearer token format
+        project_id: RevenueCat project ID (required for customers, products, entitlements, offerings, subscriptions, purchases)
+
+    Returns:
+        Iterable of DLT resources for customers, products, entitlements, offerings, purchases, subscriptions, and projects
+    """
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects() -> Iterator[Dict[str, Any]]:
+        """Get list of projects."""
+        # Get projects list
+        data = _make_request(api_key, "/projects")
+        if "items" in data:
+            for project in data["items"]:
+                project = convert_timestamps_to_iso(project, ["created_at"])
+                yield project
+
+    @dlt.resource(
+        name="customers", primary_key="id", write_disposition="merge", parallelized=True
+    )
+    def customers() -> Iterator[Dict[str, Any]]:
+        """Get list of customers with nested purchases and subscriptions."""
+        if project_id is None:
+            raise ValueError("project_id is required for customers resource")
+        endpoint = f"/projects/{project_id}/customers"
+
+        async def process_customer_batch(customer_batch):
+            """Process a batch of customers with async operations."""
+            async with aiohttp.ClientSession() as session:
+                tasks = []
+                for customer in customer_batch:
+                    task = process_customer_with_nested_resources_async(
+                        session, api_key, project_id, customer
+                    )
+                    tasks.append(task)
+
+                return await asyncio.gather(*tasks)
+
+        def process_customers_sync():
+            """Process customers in batches using asyncio."""
+            batch_size = 50  # Conservative batch size due to 60 req/min rate limit
+            current_batch = []
+
+            for customer in _paginate(api_key, endpoint):
+                current_batch.append(customer)
+
+                if len(current_batch) >= batch_size:
+                    # Process the batch asynchronously
+                    processed_customers = asyncio.run(
+                        process_customer_batch(current_batch)
+                    )
+                    for processed_customer in processed_customers:
+                        yield processed_customer
+                    current_batch = []
+
+            # Process any remaining customers in the final batch
+            if current_batch:
+                processed_customers = asyncio.run(process_customer_batch(current_batch))
+                for processed_customer in processed_customers:
+                    yield processed_customer
+
+        # Yield each processed customer
+        yield from process_customers_sync()
+
+    # Create project-dependent resources dynamically
+    project_resources = []
+    resource_names = ["products", "entitlements", "offerings"]
+    
+    for resource_name in resource_names:
+        @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+        def create_resource(resource_name=resource_name) -> Iterator[Dict[str, Any]]:
+            """Get list of project resource."""
+            yield from create_project_resource(resource_name, api_key, project_id)
+        
+        # Set the function name for better identification
+        create_resource.__name__ = resource_name
+        project_resources.append(create_resource)
+
+    return [
+        projects,
+        customers,
+        *project_resources,
+    ]

ingestr/src/revenuecat/helpers.py

@@ -0,0 +1,291 @@
+import asyncio
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+import aiohttp
+import pendulum
+import requests
+
+REVENUECAT_API_BASE = "https://api.revenuecat.com/v2"
+
+
+def _make_request(
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make a REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params or {})
+
+            # Handle rate limiting (429 Too Many Requests)
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    # Wait based on Retry-After header or exponential backoff
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                    time.sleep(wait_time)
+                    continue
+
+            response.raise_for_status()
+            return response.json()
+
+        except requests.exceptions.RequestException:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                time.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    response.raise_for_status()
+    return response.json()
+
+
+def _paginate(
+    api_key: str, endpoint: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Paginate through RevenueCat API results."""
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = _make_request(api_key, endpoint, current_params)
+
+        # Yield items from the current page
+        if "items" in data and data["items"] is not None:
+            for item in data["items"]:
+                yield item
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+
+def convert_timestamps_to_iso(
+    record: Dict[str, Any], timestamp_fields: List[str]
+) -> Dict[str, Any]:
+    """Convert timestamp fields from milliseconds to ISO format."""
+    for field in timestamp_fields:
+        if field in record and record[field] is not None:
+            # Convert from milliseconds timestamp to ISO datetime string
+            timestamp_ms = record[field]
+            dt = pendulum.from_timestamp(timestamp_ms / 1000)
+            record[field] = dt.to_iso8601_string()
+
+    return record
+
+
+async def _make_request_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make an async REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            async with session.get(
+                url, headers=headers, params=params or {}
+            ) as response:
+                # Handle rate limiting (429 Too Many Requests)
+                if response.status == 429:
+                    if attempt < max_retries:
+                        # Wait based on Retry-After header or exponential backoff
+                        retry_after = response.headers.get("Retry-After")
+                        if retry_after:
+                            wait_time = int(retry_after)
+                        else:
+                            wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                response.raise_for_status()
+                return await response.json()
+
+        except aiohttp.ClientError:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                await asyncio.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    async with session.get(url, headers=headers, params=params or {}) as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """Paginate through RevenueCat API results asynchronously."""
+    items = []
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = await _make_request_async(session, api_key, endpoint, current_params)
+
+        # Collect items from the current page
+        if "items" in data and data["items"] is not None:
+            items.extend(data["items"])
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+    return items
+
+
+async def fetch_and_process_nested_resource_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer_id: str,
+    customer: Dict[str, Any],
+    resource_name: str,
+    timestamp_fields: Optional[List[str]] = None,
+) -> None:
+    """
+    Fetch and process any nested resource for a customer asynchronously.
+
+    Args:
+        session: aiohttp ClientSession
+        api_key: RevenueCat API key
+        project_id: Project ID
+        customer_id: Customer ID
+        customer: Customer data dictionary to modify
+        resource_name: Name of the nested resource (e.g., 'purchases', 'subscriptions', 'events')
+        timestamp_fields: List of timestamp fields to convert to ISO format
+    """
+    # If resource not included in customer data, fetch separately
+    if resource_name not in customer or customer[resource_name] is None:
+        endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+        customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+
+    # Convert timestamps if fields specified
+    if (
+        timestamp_fields
+        and resource_name in customer
+        and customer[resource_name] is not None
+    ):
+        for item in customer[resource_name]:
+            convert_timestamps_to_iso(item, timestamp_fields)
+
+
+async def process_customer_with_nested_resources_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Process a customer and fetch nested resources concurrently.
+
+    Args:
+        session: aiohttp ClientSession
+        api_key: RevenueCat API key
+        project_id: Project ID
+        customer: Customer data to process
+
+    Returns:
+        Customer data with nested resources populated
+    """
+    customer_id = customer["id"]
+
+    # Convert customer timestamps
+    customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
+
+    # Define nested resources to fetch concurrently
+    nested_resources = [
+        ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
+        ("purchases", ["purchased_at", "expires_at"]),
+    ]
+
+    # Create concurrent tasks for fetching nested resources
+    tasks = []
+    for resource_name, timestamp_fields in nested_resources:
+        task = fetch_and_process_nested_resource_async(
+            session,
+            api_key,
+            project_id,
+            customer_id,
+            customer,
+            resource_name,
+            timestamp_fields,
+        )
+        tasks.append(task)
+
+    # Wait for all nested resources to be fetched
+    await asyncio.gather(*tasks)
+
+    return customer
+
+
+def create_project_resource(
+    resource_name: str,
+    api_key: str,
+    project_id: str = None,
+    timestamp_fields: List[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to create DLT resources for project-dependent endpoints.
+    
+    Args:
+        resource_name: Name of the resource (e.g., 'products', 'entitlements', 'offerings')
+        api_key: RevenueCat API key
+        project_id: RevenueCat project ID
+        timestamp_fields: List of timestamp fields to convert to ISO format
+    
+    Returns:
+        Iterator of resource data
+    """
+    if project_id is None:
+        raise ValueError(f"project_id is required for {resource_name} resource")
+    
+    endpoint = f"/projects/{project_id}/{resource_name}"
+    default_timestamp_fields = timestamp_fields or ["created_at", "updated_at"]
+    
+    for item in _paginate(api_key, endpoint):
+        item = convert_timestamps_to_iso(item, default_timestamp_fields)
+        yield item

ingestr/src/shopify/__init__.py

@@ -669,7 +669,7 @@ def shopify_source(
             params["updated_at_max"] = updated_at.end_value.isoformat()
         yield from client.get_pages("customers", params)
 
-    @dlt.resource(primary_key="id", write_disposition="append")
+    @dlt.resource(primary_key="id", write_disposition="merge")
     def events(
         created_at: dlt.sources.incremental[
             pendulum.DateTime

ingestr/src/sources.py

@@ -1054,9 +1054,10 @@ class FacebookAdsSource:
                 )
 
             # Validate breakdown type against available options from settings
-            import typing
 
-            from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
+            from ingestr.src.facebook_ads.helpers import (
+                parse_insights_table_to_source_kwargs,
+            )
 
             source_kwargs = {
                 "access_token": access_token[0],
@@ -2689,13 +2690,13 @@ class FrankfurterSource:
 
         if kwargs.get("interval_start"):
             start_date = ensure_pendulum_datetime(str(kwargs.get("interval_start")))
-            if kwargs.get("interval_end"):
-                end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
-            else:
-                end_date = pendulum.now()
         else:
-            start_date = pendulum.now()
-            end_date = pendulum.now()
+            start_date = pendulum.yesterday()
+
+        if kwargs.get("interval_end"):
+            end_date = ensure_pendulum_datetime(str(kwargs.get("interval_end")))
+        else:
+            end_date = None
 
         validate_dates(start_date=start_date, end_date=end_date)
 
@@ -3224,6 +3225,76 @@ class PinterestSource:
         ).with_resources(table)
 
 
+class FluxxSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "Fluxx takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        # Parse URI: fluxx://instance?client_id=xxx&client_secret=xxx
+        parsed_uri = urlparse(uri)
+        source_params = parse_qs(parsed_uri.query)
+
+        instance = parsed_uri.hostname
+        if not instance:
+            raise ValueError(
+                "Instance is required in the URI (e.g., fluxx://mycompany.preprod)"
+            )
+
+        client_id = source_params.get("client_id")
+        if not client_id:
+            raise ValueError("client_id in the URI is required to connect to Fluxx")
+
+        client_secret = source_params.get("client_secret")
+        if not client_secret:
+            raise ValueError("client_secret in the URI is required to connect to Fluxx")
+
+        # Parse date parameters
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date)
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date)
+
+        # Import Fluxx source
+        from ingestr.src.fluxx import fluxx_source
+
+        # Parse table specification for custom column selection
+        # Format: "resource_name:field1,field2,field3" or "resource_name"
+        resources = None
+        custom_fields = {}
+
+        if table:
+            # Handle single resource with custom fields or multiple resources
+            if ":" in table and table.count(":") == 1:
+                # Single resource with custom fields: "grant_request:id,name,amount"
+                resource_name, field_list = table.split(":", 1)
+                resource_name = resource_name.strip()
+                fields = [f.strip() for f in field_list.split(",")]
+                resources = [resource_name]
+                custom_fields[resource_name] = fields
+            else:
+                # Multiple resources or single resource without custom fields
+                # Support comma-separated list: "grant_request,user"
+                resources = [r.strip() for r in table.split(",")]
+
+        return fluxx_source(
+            instance=instance,
+            client_id=client_id[0],
+            client_secret=client_secret[0],
+            start_date=start_date,
+            end_date=end_date,
+            resources=resources,
+            custom_fields=custom_fields,
+        )
+
+
 class LinearSource:
     def handles_incrementality(self) -> bool:
         return True
@@ -3241,11 +3312,11 @@ class LinearSource:
             raise MissingValueError("api_key", "Linear")
 
         if table not in [
-            "issues", 
-            "projects", 
-            "teams", 
-            "users", 
-            "workflow_states", 
+            "issues",
+            "projects",
+            "teams",
+            "users",
+            "workflow_states",
             "cycles",
             "attachments",
             "comments",
@@ -3282,6 +3353,56 @@ class LinearSource:
         ).with_resources(table)
 
 
+class RevenueCatSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError(
+                "RevenueCat takes care of incrementality on its own, you should not provide incremental_key"
+            )
+
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "RevenueCat")
+
+        project_id = params.get("project_id")
+        if project_id is None and table != "projects":
+            raise MissingValueError("project_id", "RevenueCat")
+
+        if table not in [
+            "customers",
+            "products",
+            "entitlements",
+            "offerings",
+            "subscriptions",
+            "purchases",
+            "projects",
+        ]:
+            raise UnsupportedResourceError(table, "RevenueCat")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1).in_tz("UTC")
+
+        end_date = kwargs.get("interval_end")
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
+
+        from ingestr.src.revenuecat import revenuecat_source
+
+        return revenuecat_source(
+            api_key=api_key[0],
+            project_id=project_id[0] if project_id is not None else None,
+        ).with_resources(table)
+
+
 class ZoomSource:
     def handles_incrementality(self) -> bool:
         return True

ingestr/src/stripe_analytics/__init__.py

@@ -101,23 +101,6 @@ def incremental_stripe_source(
     initial_start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,
 ) -> Iterable[DltResource]:
-    """
-    As Stripe API does not include the "updated" key in its responses,
-    we are only able to perform incremental downloads from endpoints where all objects are uneditable.
-    This source yields the resources with incremental loading based on "append" mode.
-    You will load only the newest data without duplicating and without downloading a huge amount of data each time.
-
-    Args:
-        endpoints (tuple): A tuple of endpoint names to retrieve data from. Defaults to Stripe API endpoints with uneditable data.
-        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
-        initial_start_date (Optional[DateTime]): An optional parameter that specifies the initial value for dlt.sources.incremental.
-                            If parameter is not None, then load only data that were created after initial_start_date on the first run.
-                            Defaults to None. Format: datetime(YYYY, MM, DD).
-        end_date (Optional[DateTime]): An optional end date to limit the data retrieved.
-                  Defaults to None. Format: datetime(YYYY, MM, DD).
-    Returns:
-        Iterable[DltResource]: Resources with only that data has not yet been loaded.
-    """
     stripe.api_key = stripe_secret_key
     stripe.api_version = "2022-11-15"
     start_date_unix = (
@@ -142,6 +125,6 @@ def incremental_stripe_source(
         yield dlt.resource(
             incremental_resource,
             name=endpoint,
-            write_disposition="append",
+            write_disposition="merge",
             primary_key="id",
         )(endpoint)