ingestr 0.13.55__py3-none-any.whl → 0.13.57__py3-none-any.whl

ingestr/src/blob.py

@@ -6,6 +6,10 @@ BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
 
 
+class UnsupportedEndpointError(Exception):
+    pass
+
+
 def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     """
     parse the URI of a blob storage and
@@ -50,3 +54,23 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         return "", parts[0]
 
     return parts[0], parts[1]
+
+
+def parse_endpoint(path: str) -> str:
+    """
+    Parse the endpoint kind from the URI.
+
+    kind is a file format. one of [csv, jsonl, parquet]
+    """
+    file_extension = path.split(".")[-1]
+    if file_extension == "gz":
+        file_extension = path.split(".")[-2]
+    if file_extension == "csv":
+        endpoint = "read_csv"
+    elif file_extension == "jsonl":
+        endpoint = "read_jsonl"
+    elif file_extension == "parquet":
+        endpoint = "read_parquet"
+    else:
+        raise UnsupportedEndpointError(f"Unsupported file format: {file_extension}")
+    return endpoint

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.55"
+version = "v0.13.57"

ingestr/src/factory.py

@@ -64,6 +64,7 @@ from ingestr.src.sources import (
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
+    TrustpilotSource,
     ZendeskSource,
 )
 
@@ -165,6 +166,7 @@ class SourceDestinationFactory:
         "pipedrive": PipedriveSource,
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
+        "trustpilot": TrustpilotSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,

ingestr/src/kinesis/__init__.py

@@ -9,7 +9,7 @@ from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import StrStr, TAnyDateTime, TDataItem
 from dlt.common.utils import digest128
 
-from .helpers import get_shard_iterator, max_sequence_by_shard
+from .helpers import get_shard_iterator, get_stream_address, max_sequence_by_shard
 
 
 @dlt.resource(
@@ -42,7 +42,7 @@ def kinesis_stream(
         initial_at_timestamp (TAnyDateTime): An initial timestamp used to generate AT_TIMESTAMP or LATEST iterator when timestamp value is 0
         max_number_of_messages (int): Maximum number of messages to read in one run. Actual read may exceed that number by up to chunk_size. Defaults to None (no limit).
         milliseconds_behind_latest (int): The number of milliseconds behind the top of the shard to stop reading messages, defaults to 1000.
-        parse_json (bool): If True, assumes that messages are json strings, parses them and returns instead of `data` (otherwise). Defaults to False.
+        parse_json (bool): If True, assumes that messages are json strings, parses them and returns instead of `data` (otherwise). Defaults to True.
         chunk_size (int): The number of records to fetch at once. Defaults to 1000.
     Yields:
             Iterable[TDataItem]: Messages. Contain Kinesis envelope in `kinesis` and bytes data in `data` (if `parse_json` disabled)
@@ -65,7 +65,7 @@ def kinesis_stream(
     # so next time we request shards at AT_TIMESTAMP that is now
     resource_state["initial_at_timestamp"] = pendulum.now("UTC").subtract(seconds=1)
 
-    shards_list = kinesis_client.list_shards(StreamName=stream_name)
+    shards_list = kinesis_client.list_shards(**get_stream_address(stream_name))
     shards: List[StrStr] = shards_list["Shards"]
     while next_token := shards_list.get("NextToken"):
         shards_list = kinesis_client.list_shards(NextToken=next_token)

ingestr/src/kinesis/helpers.py

@@ -2,7 +2,7 @@ from typing import Any, Sequence, Tuple
 
 import dlt
 from dlt.common import pendulum
-from dlt.common.typing import DictStrAny, StrAny, StrStr
+from dlt.common.typing import DictStrAny, DictStrStr, StrAny, StrStr
 
 
 def get_shard_iterator(
@@ -40,7 +40,7 @@ def get_shard_iterator(
         )
 
     shard_iterator: StrStr = kinesis_client.get_shard_iterator(
-        StreamName=stream_name, ShardId=shard_id, **iterator_params
+        **get_stream_address(stream_name), ShardId=shard_id, **iterator_params
     )
     return shard_iterator["ShardIterator"], iterator_params
 
@@ -63,3 +63,20 @@ def max_sequence_by_shard(values: Sequence[StrStr]) -> StrStr:
     # we compare message sequence at shard_id
     last_value[shard_id] = max(item["seq_no"], last_value.get(shard_id, ""))
     return last_value
+
+
+def get_stream_address(stream_name: str) -> DictStrStr:
+    """
+    Return address of stream, either as StreamName or StreamARN, when applicable.
+
+    Examples:
+    - customer_events
+    - arn:aws:kinesis:eu-central-1:842404475894:stream/customer_events
+
+    https://docs.aws.amazon.com/kinesis/latest/APIReference/API_StreamDescription.html#Streams-Type-StreamDescription-StreamName
+    https://docs.aws.amazon.com/kinesis/latest/APIReference/API_StreamDescription.html#Streams-Type-StreamDescription-StreamARN
+    """
+    if stream_name.startswith("arn:"):
+        return {"StreamARN": stream_name}
+    else:
+        return {"StreamName": stream_name}

ingestr/src/sources.py

@@ -677,24 +677,33 @@ class StripeAnalyticsSource:
 
         table = table.lower()
 
-        from ingestr.src.stripe_analytics.settings import (
-            ENDPOINTS,
-            INCREMENTAL_ENDPOINTS,
-        )
+        from ingestr.src.stripe_analytics.settings import ENDPOINTS
 
-        if table in ENDPOINTS:
-            endpoint = ENDPOINTS[table]
-            from ingestr.src.stripe_analytics import stripe_source
+        endpoint = None
+        incremental = False
+        sync = False
 
-            return stripe_source(
-                endpoints=[
-                    endpoint,
-                ],
-                stripe_secret_key=api_key[0],
-            ).with_resources(endpoint)
+        table_fields = table.split(":")
+        if len(table_fields) == 1:
+            endpoint = table_fields[0]
+        elif len(table_fields) == 2:
+            endpoint = table_fields[0]
+            sync = table_fields[1] == "sync"
+        elif len(table_fields) == 3:
+            endpoint = table_fields[0]
+            sync = table_fields[1] == "sync"
+            incremental = table_fields[2] == "incremental"
+        else:
+            raise ValueError(
+                "Invalid Stripe table format. Expected: stripe:<endpoint> or stripe:<endpoint>:<sync> or stripe:<endpoint>:<sync>:<incremental>"
+            )
+
+        if incremental and not sync:
+            raise ValueError(
+                "incremental loads must be used with sync loading"
+            )
 
-        elif table in INCREMENTAL_ENDPOINTS:
-            endpoint = INCREMENTAL_ENDPOINTS[table]
+        if incremental:
             from ingestr.src.stripe_analytics import incremental_stripe_source
 
             def nullable_date(date_str: Optional[str]):
@@ -702,6 +711,7 @@ class StripeAnalyticsSource:
                     return ensure_pendulum_datetime(date_str)
                 return None
 
+            endpoint = ENDPOINTS[endpoint]
             return incremental_stripe_source(
                 endpoints=[
                     endpoint,
@@ -710,6 +720,26 @@ class StripeAnalyticsSource:
                 initial_start_date=nullable_date(kwargs.get("interval_start", None)),
                 end_date=nullable_date(kwargs.get("interval_end", None)),
             ).with_resources(endpoint)
+        else:
+            endpoint = ENDPOINTS[endpoint]
+            if sync:
+                from ingestr.src.stripe_analytics import stripe_source
+
+                return stripe_source(
+                    endpoints=[
+                        endpoint,
+                    ],
+                    stripe_secret_key=api_key[0],
+                ).with_resources(endpoint)
+            else:
+                from ingestr.src.stripe_analytics import async_stripe_source
+
+                return async_stripe_source(
+                    endpoints=[
+                        endpoint,
+                    ],
+                    stripe_secret_key=api_key[0],
+                ).with_resources(endpoint)
 
         raise ValueError(
             f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
@@ -1362,17 +1392,25 @@ class S3Source:
             secret=secret_access_key[0],
         )
 
-        file_extension = path_to_file.split(".")[-1]
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
+        endpoint: Optional[str] = None
+        if "#" in table:
+            _, endpoint = table.split("#")
+            if endpoint not in ["csv", "jsonl", "parquet"]:
+                raise ValueError(
+                    "S3 Source only supports specific formats files: csv, jsonl, parquet"
+                )
+            endpoint = f"read_{endpoint}"
         else:
-            raise ValueError(
-                "S3 Source only supports specific formats files: csv, jsonl, parquet"
-            )
+            try:
+                endpoint = blob.parse_endpoint(path_to_file)
+            except blob.UnsupportedEndpointError:
+                raise ValueError(
+                    "S3 Source only supports specific formats files: csv, jsonl, parquet"
+                )
+            except Exception as e:
+                raise ValueError(
+                    f"Failed to parse endpoint from path: {path_to_file}"
+                ) from e
 
         from ingestr.src.filesystem import readers
 
@@ -1844,17 +1882,16 @@ class GCSSource:
             token=credentials,
         )
 
-        file_extension = path_to_file.split(".")[-1]
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
-        else:
+        try:
+            endpoint = blob.parse_endpoint(path_to_file)
+        except blob.UnsupportedEndpointError:
             raise ValueError(
-                "GCS Source only supports specific formats files: csv, jsonl, parquet"
+                "S3 Source only supports specific formats files: csv, jsonl, parquet"
             )
+        except Exception as e:
+            raise ValueError(
+                f"Failed to parse endpoint from path: {path_to_file}"
+            ) from e
 
         from ingestr.src.filesystem import readers
 
@@ -2392,6 +2429,47 @@ class FreshdeskSource:
         ).with_resources(table)
 
 
+class TrustpilotSource:
+    # trustpilot://<business_unit_id>?api_key=<api_key>
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        business_unit_id = parsed_uri.netloc
+        params = parse_qs(parsed_uri.query)
+
+        if not business_unit_id:
+            raise MissingValueError("business_unit_id", "Trustpilot")
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Trustpilot")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = ensure_pendulum_datetime("2000-01-01").in_tz("UTC").isoformat()
+        else:
+            start_date = ensure_pendulum_datetime(start_date).in_tz("UTC").isoformat()
+
+        end_date = kwargs.get("interval_end")
+
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC").isoformat()
+
+        if table not in ["reviews"]:
+            raise UnsupportedResourceError(table, "Trustpilot")
+
+        from ingestr.src.trustpilot import trustpilot_source
+
+        return trustpilot_source(
+            business_unit_id=business_unit_id,
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)
+
+
 class PhantombusterSource:
     def handles_incrementality(self) -> bool:
         return True
@@ -2622,18 +2700,15 @@ class SFTPSource:
         else:
             file_glob = f"/{table}"
 
-        file_extension = table.split(".")[-1].lower()
-        endpoint: str
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
-        else:
+        try:
+            endpoint = blob.parse_endpoint(table)
+        except blob.UnsupportedEndpointError:
             raise ValueError(
-                "FTPServer Source only supports specific file formats: csv, jsonl, parquet."
+                "SFTP Source only supports specific formats files: csv, jsonl, parquet"
             )
+        except Exception as e:
+            raise ValueError(f"Failed to parse endpoint from path: {table}") from e
+
         from ingestr.src.filesystem import readers
 
         dlt_source_resource = readers(bucket_url, fs, file_glob)

ingestr/src/stripe_analytics/__init__.py

@@ -7,7 +7,12 @@ import stripe
 from dlt.sources import DltResource
 from pendulum import DateTime
 
-from .helpers import pagination, transform_date
+from .helpers import (
+    async_parallel_pagination,
+    pagination,
+    parallel_pagination,
+    transform_date,
+)
 
 
 @dlt.source(max_table_nesting=0)
@@ -50,6 +55,86 @@ def stripe_source(
         )(endpoint)
 
 
+@dlt.source(max_table_nesting=0)
+def parallel_stripe_source(
+    endpoints: Tuple[str, ...],
+    stripe_secret_key: str = dlt.secrets.value,
+    start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+    max_workers: int = 12,
+) -> Iterable[DltResource]:
+    """
+    Retrieves data from the Stripe API for the specified endpoints using parallel pagination.
+
+    This source divides the date range across multiple workers to fetch data in parallel,
+    which can significantly speed up data retrieval for large date ranges.
+
+    Args:
+        endpoints (Tuple[str, ...]): A tuple of endpoint names to retrieve data from.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date (Optional[DateTime]): An optional start date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Required for parallel processing.
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Required for parallel processing.
+        max_workers (int): Maximum number of worker threads for parallel fetching. Defaults to 4.
+
+    Returns:
+        Iterable[DltResource]: Resources with data that was created during the period greater than or equal to 'start_date' and less than 'end_date'.
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+
+    def parallel_stripe_resource(
+        endpoint: str,
+    ) -> Generator[Dict[Any, Any], Any, None]:
+        yield from parallel_pagination(endpoint, start_date, end_date, max_workers)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            parallel_stripe_resource,
+            name=endpoint,
+            write_disposition="replace",
+        )(endpoint)
+
+
+@dlt.source(max_table_nesting=0)
+def async_stripe_source(
+    endpoints: Tuple[str, ...],
+    stripe_secret_key: str = dlt.secrets.value,
+    start_date: Optional[DateTime] = None,
+    end_date: Optional[DateTime] = None,
+    max_workers: int = 40,
+    rate_limit_delay: float = 0.03,
+) -> Iterable[DltResource]:
+    """
+    ULTRA-FAST async Stripe source optimized for maximum speed and throughput.
+
+    WARNING: Returns data in RANDOM ORDER for maximum performance.
+    Uses aggressive concurrency and minimal delays to maximize API throughput.
+
+    Args:
+        endpoints (Tuple[str, ...]): A tuple of endpoint names to retrieve data from.
+        stripe_secret_key (str): The API access token for authentication. Defaults to the value in the `dlt.secrets` object.
+        start_date (Optional[DateTime]): An optional start date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to 2010-01-01.
+        end_date (Optional[DateTime]): An optional end date to limit the data retrieved. Format: datetime(YYYY, MM, DD). Defaults to today.
+        max_workers (int): Maximum number of concurrent async tasks. Defaults to 40 for maximum speed.
+        rate_limit_delay (float): Minimal delay between requests. Defaults to 0.03 seconds.
+
+    Returns:
+        Iterable[DltResource]: Resources with data in RANDOM ORDER (optimized for speed).
+    """
+    stripe.api_key = stripe_secret_key
+    stripe.api_version = "2022-11-15"
+
+    async def async_stripe_resource(endpoint: str):
+        yield async_parallel_pagination(endpoint, max_workers, rate_limit_delay)
+
+    for endpoint in endpoints:
+        yield dlt.resource(
+            async_stripe_resource,
+            name=endpoint,
+            write_disposition="replace",
+        )(endpoint)
+
+
 @dlt.source
 def incremental_stripe_source(
     endpoints: Tuple[str, ...],

ingestr/src/stripe_analytics/helpers.py

@@ -1,6 +1,10 @@
 """Stripe analytics source helpers"""
 
-from typing import Any, Dict, Iterable, Optional, Union
+import asyncio
+import math
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timedelta
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 import stripe
 from dlt.common import pendulum
@@ -39,6 +43,300 @@ def pagination(
             break
 
 
+def parallel_pagination(
+    endpoint: str,
+    start_date: Optional[Any] = None,
+    end_date: Optional[Any] = None,
+    max_workers: int = 4,
+) -> Iterable[TDataItem]:
+    """
+    Retrieves data from an endpoint with parallel pagination by dividing date ranges across workers.
+
+    Args:
+        endpoint (str): The endpoint to retrieve data from.
+        start_date (Optional[Any]): An optional start date to limit the data retrieved. Defaults to 2010-01-01 if None.
+        end_date (Optional[Any]): An optional end date to limit the data retrieved. Defaults to today if None.
+        max_workers (int): Maximum number of worker threads to use for parallel fetching. Defaults to 4.
+
+    Returns:
+        Iterable[TDataItem]: Data items retrieved from the endpoint.
+    """
+    # Set default date range if not provided: 2010 to today
+    if not start_date:
+        start_date = pendulum.datetime(2010, 1, 1)
+    if not end_date:
+        end_date = pendulum.now()
+
+    # Convert dates to timestamps for processing
+    start_ts = transform_date(start_date)
+    end_ts = transform_date(end_date)
+
+    # If date range is very small, use sequential pagination
+    date_range_days = (end_ts - start_ts) / (24 * 60 * 60)
+    if date_range_days < 30:  # Less than 30 days
+        yield from pagination(endpoint, start_date, end_date)
+        return
+
+    # Create time chunks with larger chunks for 2010s (less data expected)
+    time_chunks = _create_adaptive_time_chunks(start_ts, end_ts, max_workers)
+
+    # Use ThreadPoolExecutor to fetch data in parallel and yield as soon as ready
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        # Submit all tasks
+        future_to_chunk = {
+            executor.submit(
+                _fetch_chunk_data_streaming, endpoint, chunk_start, chunk_end
+            ): (chunk_start, chunk_end)
+            for chunk_start, chunk_end in time_chunks
+        }
+
+        # MAXIMUM SPEED - Yield results immediately as they complete
+        for future in as_completed(future_to_chunk):
+            chunk_start, chunk_end = future_to_chunk[future]
+            try:
+                chunk_data = future.result()
+                # Yield all batches from this chunk immediately - NO ORDERING
+                for batch in chunk_data:
+                    yield batch
+
+            except Exception as exc:
+                print(f"Chunk {chunk_start}-{chunk_end} generated an exception: {exc}")
+                raise exc
+
+
+def _create_time_chunks(start_ts: int, end_ts: int, num_chunks: int) -> List[tuple]:
+    """
+    Divide a time range into equal chunks for parallel processing.
+
+    Args:
+        start_ts (int): Start timestamp
+        end_ts (int): End timestamp
+        num_chunks (int): Number of chunks to create
+
+    Returns:
+        List[tuple]: List of (chunk_start, chunk_end) timestamp pairs
+    """
+    total_duration = end_ts - start_ts
+    chunk_duration = math.ceil(total_duration / num_chunks)
+
+    chunks = []
+    current_start = start_ts
+
+    for i in range(num_chunks):
+        current_end = min(current_start + chunk_duration, end_ts)
+        if current_start < end_ts:
+            chunks.append((current_start, current_end))
+        current_start = current_end
+
+        if current_start >= end_ts:
+            break
+
+    return chunks
+
+
+def _create_adaptive_time_chunks(
+    start_ts: int, end_ts: int, max_workers: int
+) -> List[tuple]:
+    """
+    Create time chunks with adaptive sizing - larger chunks for 2010s (less data expected).
+
+    Args:
+        start_ts (int): Start timestamp
+        end_ts (int): End timestamp
+        max_workers (int): Maximum number of workers
+
+    Returns:
+        List[tuple]: List of (chunk_start, chunk_end) timestamp pairs
+    """
+    chunks = []
+
+    # Key timestamps
+    year_2020_ts = int(pendulum.datetime(2020, 1, 1).timestamp())
+    year_2015_ts = int(pendulum.datetime(2015, 1, 1).timestamp())
+
+    current_start = start_ts
+
+    # Handle 2010-2015: Large chunks (2-3 year periods)
+    if current_start < year_2015_ts:
+        chunk_end = min(year_2015_ts, end_ts)
+        if current_start < chunk_end:
+            # Split 2010-2015 into 2-3 chunks max
+            pre_2015_chunks = _create_time_chunks(
+                current_start, chunk_end, min(3, max_workers)
+            )
+            chunks.extend(pre_2015_chunks)
+        current_start = chunk_end
+
+    # Handle 2015-2020: Medium chunks (6 month to 1 year periods)
+    if current_start < year_2020_ts and current_start < end_ts:
+        chunk_end = min(year_2020_ts, end_ts)
+        if current_start < chunk_end:
+            # Split 2015-2020 into smaller chunks
+            duration_2015_2020 = chunk_end - current_start
+            years_2015_2020 = duration_2015_2020 / (365 * 24 * 60 * 60)
+            num_chunks_2015_2020 = min(
+                max_workers, max(2, int(years_2015_2020 * 2))
+            )  # ~6 months per chunk
+
+            pre_2020_chunks = _create_time_chunks(
+                current_start, chunk_end, num_chunks_2015_2020
+            )
+            chunks.extend(pre_2020_chunks)
+        current_start = chunk_end
+
+    if current_start < end_ts:
+        # Split post-2020 data into daily chunks for maximum granularity
+        current_chunk_start = current_start
+        while current_chunk_start < end_ts:
+            # Calculate end of current day
+            current_date = datetime.fromtimestamp(current_chunk_start)
+            next_day = current_date + timedelta(days=1)
+            chunk_end = min(int(next_day.timestamp()), end_ts)
+
+            chunks.append((current_chunk_start, chunk_end))
+            current_chunk_start = chunk_end
+
+    return chunks
+
+
+def _fetch_chunk_data_streaming(
+    endpoint: str, start_ts: int, end_ts: int
+) -> List[List[TDataItem]]:
+    """
+    Fetch data for a specific time chunk using sequential pagination with memory-efficient approach.
+
+    Args:
+        endpoint (str): The Stripe endpoint to fetch from
+        start_ts (int): Start timestamp for this chunk
+        end_ts (int): End timestamp for this chunk
+
+    Returns:
+        List[List[TDataItem]]: List of batches of data items
+    """
+    # For streaming, we still need to collect the chunk data to maintain order
+    # but we can optimize by not holding all data in memory at once
+    print(
+        f"Fetching chunk {datetime.fromtimestamp(start_ts).strftime('%Y-%m-%d')}-{datetime.fromtimestamp(end_ts).strftime('%Y-%m-%d')}"
+    )
+    chunk_data = []
+    batch_count = 0
+
+    for batch in pagination(endpoint, start_ts, end_ts):
+        chunk_data.append(batch)
+        print(
+            f"Processed {batch_count} batches for chunk {datetime.fromtimestamp(start_ts).strftime('%Y-%m-%d')}-{datetime.fromtimestamp(end_ts).strftime('%Y-%m-%d')}"
+        )
+        batch_count += 1
+
+    return chunk_data
+
+
+async def async_pagination(
+    endpoint: str, start_date: Optional[Any] = None, end_date: Optional[Any] = None
+) -> Iterable[TDataItem]:
+    """
+    Async version of pagination that retrieves data from an endpoint with pagination.
+
+    Args:
+        endpoint (str): The endpoint to retrieve data from.
+        start_date (Optional[Any]): An optional start date to limit the data retrieved. Defaults to None.
+        end_date (Optional[Any]): An optional end date to limit the data retrieved. Defaults to None.
+
+    Returns:
+        Iterable[TDataItem]: Data items retrieved from the endpoint.
+    """
+    starting_after = None
+    while True:
+        response = await stripe_get_data_async(
+            endpoint,
+            start_date=start_date,
+            end_date=end_date,
+            starting_after=starting_after,
+        )
+
+        if len(response["data"]) > 0:
+            starting_after = response["data"][-1]["id"]
+        yield response["data"]
+
+        if not response["has_more"]:
+            break
+
+
+async def async_parallel_pagination(
+    endpoint: str,
+    max_workers: int = 8,
+    rate_limit_delay: float = 5,
+) -> Iterable[TDataItem]:
+    """
+    ULTRA-FAST async parallel pagination - yields data in random order for maximum speed.
+    No ordering constraints - pure performance optimization.
+
+    Args:
+        endpoint (str): The endpoint to retrieve data from.
+        start_date (Optional[Any]): An optional start date to limit the data retrieved. Defaults to 2010-01-01 if None.
+        end_date (Optional[Any]): An optional end date to limit the data retrieved. Defaults to today if None.
+        max_workers (int): Maximum number of concurrent async tasks. Defaults to 8 for balanced speed/rate limit respect.
+        rate_limit_delay (float): Minimal delay between requests. Defaults to 5 seconds.
+
+    Returns:
+        Iterable[TDataItem]: Data items retrieved from the endpoint (RANDOM ORDER FOR SPEED).
+    """
+
+    start_date = pendulum.datetime(2010, 1, 1)
+    end_date = pendulum.now()
+    start_ts = transform_date(start_date)
+    end_ts = transform_date(end_date)
+
+    # Create time chunks with larger chunks for 2010s (less data expected)
+    time_chunks = _create_adaptive_time_chunks(start_ts, end_ts, max_workers)
+
+    # Use asyncio semaphore to control concurrency and respect rate limits
+    semaphore = asyncio.Semaphore(max_workers)
+
+    async def fetch_chunk_with_semaphore(chunk_start: int, chunk_end: int):
+        async with semaphore:
+            await asyncio.sleep(rate_limit_delay)
+            return await _fetch_chunk_data_async_fast(endpoint, chunk_start, chunk_end)
+
+    # Create all tasks
+    tasks = [
+        fetch_chunk_with_semaphore(chunk_start, chunk_end)
+        for chunk_start, chunk_end in time_chunks
+    ]
+
+    for coro in asyncio.as_completed(tasks):
+        try:
+            chunk_data = await coro
+
+            for batch in chunk_data:
+                yield batch
+
+        except Exception as exc:
+            print(f"Async chunk processing generated an exception: {exc}")
+            raise exc
+
+
+async def _fetch_chunk_data_async_fast(
+    endpoint: str, start_ts: int, end_ts: int
+) -> List[List[TDataItem]]:
+    """
+    ULTRA-FAST async chunk fetcher - no metadata overhead, direct data return.
+
+    Args:
+        endpoint (str): The Stripe endpoint to fetch from
+        start_ts (int): Start timestamp for this chunk
+        end_ts (int): End timestamp for this chunk
+
+    Returns:
+        List[List[TDataItem]]: Raw batches with zero overhead
+    """
+    chunk_data = []
+    async for batch in async_pagination(endpoint, start_ts, end_ts):
+        chunk_data.append(batch)
+
+    return chunk_data
+
+
 def transform_date(date: Union[str, DateTime, int]) -> int:
     if isinstance(date, str):
         date = pendulum.from_format(date, "%Y-%m-%dT%H:%M:%SZ")
@@ -66,3 +364,45 @@ def stripe_get_data(
         created={"gte": start_date, "lt": end_date}, limit=100, **kwargs
     )
     return dict(resource_dict)
+
+
+async def stripe_get_data_async(
+    resource: str,
+    start_date: Optional[Any] = None,
+    end_date: Optional[Any] = None,
+    **kwargs: Any,
+) -> Dict[Any, Any]:
+    """Async version of stripe_get_data"""
+    if start_date:
+        start_date = transform_date(start_date)
+    if end_date:
+        end_date = transform_date(end_date)
+
+    if resource == "Subscription":
+        kwargs.update({"status": "all"})
+
+    import asyncio
+
+    from stripe import RateLimitError
+
+    max_retries = 50
+    retry_count = 0
+    max_wait_time_ms = 10000
+
+    while retry_count < max_retries:
+        try:
+            resource_dict = await getattr(stripe, resource).list_async(
+                created={"gte": start_date, "lt": end_date}, limit=100, **kwargs
+            )
+            return dict(resource_dict)
+        except RateLimitError:
+            retry_count += 1
+            if retry_count < max_retries:
+                wait_time = min(2**retry_count * 0.001, max_wait_time_ms)
+                await asyncio.sleep(wait_time)
+            else:
+                # Re-raise the last exception if we've exhausted retries
+                print(f"✗ Failed to fetch {resource} after {max_retries} retries")
+                raise
+
+    return dict(resource_dict)

ingestr/src/stripe_analytics/settings.py

@@ -11,6 +11,7 @@ ENDPOINTS = {
     "checkoutsession": "CheckoutSession",
     "checkout_session": "CheckoutSession",
     "coupon": "Coupon",
+    "charge": "Charge",
     "customer": "Customer",
     "dispute": "Dispute",
     "paymentintent": "PaymentIntent",
@@ -52,22 +53,14 @@ ENDPOINTS = {
     "top_up": "Topup",
     "webhookendpoint": "WebhookEndpoint",
     "webhook_endpoint": "WebhookEndpoint",
-}
-# possible incremental endpoints
-INCREMENTAL_ENDPOINTS = {
-    "applicationfee": "ApplicationFee",
-    "application_fee": "ApplicationFee",
-    "balancetransaction": "BalanceTransaction",
-    "balance_transaction": "BalanceTransaction",
-    "charge": "Charge",
-    "creditnote": "CreditNote",
-    "credit_note": "CreditNote",
-    "event": "Event",
     "invoice": "Invoice",
     "invoiceitem": "InvoiceItem",
     "invoice_item": "InvoiceItem",
     "invoicelineitem": "InvoiceLineItem",
     "invoice_line_item": "InvoiceLineItem",
-    "setupattempt": "SetupAttempt",
-    "setup_attempt": "SetupAttempt",
+    "balancetransaction": "BalanceTransaction",
+    "balance_transaction": "BalanceTransaction",
+    "creditnote": "CreditNote",
+    "credit_note": "CreditNote",
+    "event": "Event",
 }

ingestr/src/trustpilot/__init__.py

@@ -0,0 +1,48 @@
+"""Trustpilot source for ingesting reviews."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .client import TrustpilotClient
+
+
+@dlt.source()
+def trustpilot_source(
+    business_unit_id: str,
+    start_date: str,
+    end_date: str | None,
+    api_key: str,
+    per_page: int = 1000,
+) -> Iterable[DltResource]:
+    """Return resources for Trustpilot."""
+
+    client = TrustpilotClient(api_key=api_key)
+
+    @dlt.resource(name="reviews", write_disposition="merge", primary_key="id")
+    def reviews(
+        dateTime=(
+            dlt.sources.incremental(
+                "updated_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Generator[Dict[str, Any], None, None]:
+        if end_date is None:
+            end_dt = pendulum.now(tz="UTC").isoformat()
+        else:
+            end_dt = dateTime.end_value
+        start_dt = dateTime.last_value
+        yield from client.paginated_reviews(
+            business_unit_id=business_unit_id,
+            per_page=per_page,
+            updated_since=start_dt,
+            end_date=end_dt,
+        )
+
+    yield reviews

ingestr/src/trustpilot/client.py

@@ -0,0 +1,48 @@
+"""Simple Trustpilot API client."""
+
+from typing import Any, Dict, Iterable
+
+import pendulum
+from dlt.sources.helpers import requests
+
+
+class TrustpilotClient:
+    """Client for the Trustpilot public API."""
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+        self.base_url = "https://api.trustpilot.com/v1"
+
+    def _get(self, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        params = dict(params)
+        params["apikey"] = self.api_key
+        response = requests.get(f"{self.base_url}{endpoint}", params=params)
+        response.raise_for_status()
+        return response.json()
+
+    def paginated_reviews(
+        self,
+        business_unit_id: str,
+        updated_since: str,
+        end_date: str,
+        per_page: int = 1000,
+    ) -> Iterable[Dict[str, Any]]:
+        page = 1
+        while True:
+            params: Dict[str, Any] = {"perPage": per_page, "page": page}
+            if updated_since:
+                params["updatedSince"] = updated_since
+            data = self._get(f"/business-units/{business_unit_id}/reviews", params)
+            reviews = data.get("reviews", data)
+            if not reviews:
+                break
+            for review in reviews:
+                end_date_dt = pendulum.parse(end_date)
+                review["updated_at"] = review["updatedAt"]
+                review_dt = pendulum.parse(review["updated_at"])
+                if review_dt > end_date_dt:  # type: ignore
+                    continue
+                yield review
+            if len(reviews) < per_page:
+                break
+            page += 1

{ingestr-0.13.55.dist-info → ingestr-0.13.57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.55
+Version: 0.13.57
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -27,7 +27,6 @@ Requires-Dist: asynch==0.2.4
 Requires-Dist: attrs==25.1.0
 Requires-Dist: backoff==2.2.1
 Requires-Dist: bcrypt==4.3.0
-Requires-Dist: beautifulsoup4==4.13.3
 Requires-Dist: boto3==1.37.1
 Requires-Dist: botocore==1.37.1
 Requires-Dist: cachetools==5.5.2
@@ -156,7 +155,6 @@ Requires-Dist: python-quickbooks==0.9.2
 Requires-Dist: pytz==2025.1
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: rauth==0.7.3
-Requires-Dist: redshift-connector==2.1.5
 Requires-Dist: requests-file==2.1.0
 Requires-Dist: requests-oauthlib==1.3.1
 Requires-Dist: requests-toolbelt==1.0.0
@@ -168,7 +166,6 @@ Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2025.3.2
 Requires-Dist: s3transfer==0.11.3
-Requires-Dist: scramp==1.4.5
 Requires-Dist: semver==3.0.4
 Requires-Dist: setuptools==75.8.2
 Requires-Dist: shellingham==1.5.4
@@ -180,7 +177,6 @@ Requires-Dist: smmap==5.0.2
 Requires-Dist: snowflake-connector-python==3.14.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sortedcontainers==2.4.0
-Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
 Requires-Dist: sqlalchemy-cratedb==0.42.0.dev2
 Requires-Dist: sqlalchemy-hana==2.0.0

{ingestr-0.13.55.dist-info → ingestr-0.13.57.dist-info}/RECORD

@@ -1,17 +1,17 @@
 ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
 ingestr/main.py,sha256=GkC1hdq8AVGrvolc95zMfjmibI95p2pmFkbgCOVf-Og,26311
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
-ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=bdi0-mZhnHheYgs6WuEb8p-RIk_RFAXRCF9HalRfV0k,21
+ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
+ingestr/src/buildinfo.py,sha256=mMPkVD8bGtndOM5rdoqMRQZCk_zPuHM2EG9vRNh5cIk,21
 ingestr/src/destinations.py,sha256=TcxM2rcwHfgMMP6U0yRNcfWKlEzkBbZbqCIDww7lkTY,16882
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=mcjgbmrZr6TvP9fCMQxo-aMGcrb2PqToRcSLp5nldww,6138
+ingestr/src/factory.py,sha256=R7KzGRQ9tYZ_N-daD9OtnEp0K-DrsP8bUyXWdv4LV4A,6200
 ingestr/src/filters.py,sha256=LLecXe9QkLFkFLUZ92OXNdcANr1a8edDxrflc2ko_KA,1452
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=3ozLt9lhhNANspfjA2vb8u6qjgBJezH8QBV1XKqT1fg,94124
+ingestr/src/sources.py,sha256=dtr5sXF2ecO54eJR1Mu-dH9YgSxAVx93gjfSwJISI00,96867
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -75,8 +75,8 @@ ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
-ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
-ingestr/src/kinesis/helpers.py,sha256=aF0GCDKSectaaW8XPdERY_6bUs0ky19dcBs24ZFn-o0,2473
+ingestr/src/kinesis/__init__.py,sha256=YretSz4F28tbkcPhd55mBp2Xk7XE9unyWx0nmvl8iEc,6235
+ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0s,3152
 ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
 ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
@@ -117,13 +117,15 @@ ingestr/src/solidgate/__init__.py,sha256=JdaXvAu5QGuf9-FY294vwCQCEmfrqIld9oqbzqC
 ingestr/src/solidgate/helpers.py,sha256=oePEc9nnvmN3IaKrfJCvyKL79xdGM0-gRTN3-8tY4Fc,4952
 ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
-ingestr/src/stripe_analytics/__init__.py,sha256=j3Vmvo8G75fJJIF4rUnpGliGTpYQZt372wo-AjGImYs,4581
-ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
-ingestr/src/stripe_analytics/settings.py,sha256=ZahhZg3Sq2KnvnDcfSaXO494Csy3tElBDEHnvA1AVmA,2461
+ingestr/src/stripe_analytics/__init__.py,sha256=g2miuPexUcPEEMzmPQZqxEaQ0Q8YjUAkOvKaLn3KC-c,8219
+ingestr/src/stripe_analytics/helpers.py,sha256=8in6k1ndTon7xNh8QPDqThBWvKY9XQrmrJXveAOA6R4,13858
+ingestr/src/stripe_analytics/settings.py,sha256=xt1-ljwP4nLTNUa8l3KwFbtK8FtQHgHpzGF5uPKfRsw,2246
 ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
 ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
 ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
 ingestr/src/tiktok_ads/tiktok_helpers.py,sha256=jmWHvZzN1Vt_PWrJkgq5a2wIwon-OBEzXoZx0jEy-74,3905
+ingestr/src/trustpilot/__init__.py,sha256=ofhjep4qRPIi8q41qc97QVex8UbWF-Fd7gUsqeQlQX8,1279
+ingestr/src/trustpilot/client.py,sha256=zKYt5C7nrR83Id0KN49EPmtml8MEtlSPlAosEFU3VXY,1616
 ingestr/src/zendesk/__init__.py,sha256=tmJ_jdb6kpwmEKpcv6Im71-bOZI6h-Tcofe18OH4I24,17762
 ingestr/src/zendesk/settings.py,sha256=Vdj706nTJFQ-3KH4nO97iYCQuba3dV3E9gfnmLK6xwU,2294
 ingestr/src/zendesk/helpers/__init__.py,sha256=YTJejCiUjfIcsj9FrkY0l-JGYDI7RRte1Ydq5FDH_0c,888
@@ -139,8 +141,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
-ingestr-0.13.55.dist-info/METADATA,sha256=WNMM4qLCTDJg4xUnYNefHffB6vidRl4xopoBaaux-FM,15131
-ingestr-0.13.55.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.55.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.55.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.55.dist-info/RECORD,,
+ingestr-0.13.57.dist-info/METADATA,sha256=k5LnyrqN3QlhkaGqKGtQBD370oNkTTkoEWDAhlCq-3M,14993
+ingestr-0.13.57.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.57.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.57.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.57.dist-info/RECORD,,