ingestr 0.13.55__tar.gz → 0.13.56__tar.gz

{ingestr-0.13.55 → ingestr-0.13.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.55
+Version: 0.13.56
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.55 → ingestr-0.13.56}/docs/.vitepress/config.mjs

@@ -123,6 +123,7 @@ export default defineConfig({
               },
               { text: "Frankfurter", link: "/supported-sources/frankfurter.md" },
               { text: "Freshdesk", link: "/supported-sources/freshdesk.md" },
+              { text: "Trustpilot", link: "/supported-sources/trustpilot.md" },
               { text: "Google Cloud Storage (GCS)", link: "/supported-sources/gcs.md" },
               { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
               { text: "Google Ads", link: "/supported-sources/google-ads.md" },

{ingestr-0.13.55 → ingestr-0.13.56}/docs/supported-sources/s3.md

@@ -134,3 +134,49 @@ The `<file-glob-pattern>` in the `--source-table` argument allows for flexible f
 | `bucket/myFolder/**/*.jsonl`                | Retrieves all JSONL files recursively from the `myFolder` directory and its subdirectories in `s3://bucket`. |
 | `bucket/myFolder/mySubFolder/users.parquet` | Retrieves the specific `users.parquet` file from the `myFolder/mySubFolder/` path in `s3://bucket`.        |
 | `bucket/employees.jsonl`                    | Retrieves the `employees.jsonl` file located at the root level of the `s3://bucket`.                       |
+
+
+### Working with compressed files
+
+`ingestr` automatically detects and handles gzipped files in your S3 bucket. You can load data from compressed files with the `.gz` extension without any additional configuration.
+
+For example, to load data from a gzipped CSV file:
+
+```sh
+ingestr ingest \
+    --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table 'my_bucket/logs/event-data.csv.gz' \
+    --dest-uri duckdb:///compressed_data.duckdb \
+    --dest-table 'logs.events'
+```
+
+You can also use glob patterns to load multiple compressed files:
+
+```sh
+ingestr ingest \
+    --source-uri 's3://?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table 'my_bucket/logs/**/*.csv.gz' \
+    --dest-uri duckdb:///compressed_data.duckdb \
+    --dest-table 'logs.events'
+```
+
+### File type hinting
+
+If your files are properly encoded but lack the correct file extension (CSV, JSONL, or Parquet), you can provide a file type hint to inform `ingestr` about the format of the files. This is done by appending a fragment identifier (`#format`) to the end of the path in your `--source-table` parameter.
+
+For example, if you have JSONL-formatted log files stored in S3 with a non-standard extension:
+
+```
+--source-table "my_bucket/logs/event-data#jsonl"
+```
+
+This tells `ingestr` to process the files as JSONL, regardless of their actual extension.
+
+Supported format hints include:
+- `#csv` - For comma-separated values files
+- `#jsonl` - For line-delimited JSON files
+- `#parquet` - For Parquet format files
+
+::: tip
+File type hinting works with `gzip` compressed files as well.
+:::

ingestr-0.13.56/docs/supported-sources/trustpilot.md

@@ -0,0 +1,35 @@
+# Trustpilot
+
+[Trustpilot](https://www.trustpilot.com/) provides a platform for collecting and
+sharing customer reviews.
+
+ingestr supports Trustpilot as a source.
+
+## URI format
+
+The URI format for Trustpilot is:
+
+```
+trustpilot://<business_unit_id>?api_key=<api_key>
+```
+
+URI parameters:
+- `api_key`: Your Trustpilot API key.
+- `business_unit_id`: Identifier of the business unit whose reviews you want to fetch.
+
+## Example usage
+
+Assuming your `business_unit_id` is `123` and your API key is `key_abc`, you can ingest reviews into DuckDB using:
+
+```bash
+ingestr ingest --source-uri 'trustpilot://123?api_key=key_abc' --source-table 'reviews' --dest-uri duckdb:///trustpilot.duckdb --dest-table 'dest.reviews'
+```
+
+## Tables
+
+Currently the Trustpilot source exposes the following table:
+
+| Name    | Description                                 |
+| ------- | ------------------------------------------- |
+| reviews | Customer reviews for the specified business |
+

{ingestr-0.13.55 → ingestr-0.13.56}/ingestr/src/blob.py

@@ -6,6 +6,10 @@ BucketName: TypeAlias = str
 FileGlob: TypeAlias = str
 
 
+class UnsupportedEndpointError(Exception):
+    pass
+
+
 def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
     """
     parse the URI of a blob storage and
@@ -50,3 +54,23 @@ def parse_uri(uri: ParseResult, table: str) -> Tuple[BucketName, FileGlob]:
         return "", parts[0]
 
     return parts[0], parts[1]
+
+
+def parse_endpoint(path: str) -> str:
+    """
+    Parse the endpoint kind from the URI.
+
+    kind is a file format. one of [csv, jsonl, parquet]
+    """
+    file_extension = path.split(".")[-1]
+    if file_extension == "gz":
+        file_extension = path.split(".")[-2]
+    if file_extension == "csv":
+        endpoint = "read_csv"
+    elif file_extension == "jsonl":
+        endpoint = "read_jsonl"
+    elif file_extension == "parquet":
+        endpoint = "read_parquet"
+    else:
+        raise UnsupportedEndpointError(f"Unsupported file format: {file_extension}")
+    return endpoint

ingestr-0.13.56/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.56"

{ingestr-0.13.55 → ingestr-0.13.56}/ingestr/src/factory.py

@@ -64,6 +64,7 @@ from ingestr.src.sources import (
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
+    TrustpilotSource,
     ZendeskSource,
 )
 
@@ -165,6 +166,7 @@ class SourceDestinationFactory:
         "pipedrive": PipedriveSource,
         "frankfurter": FrankfurterSource,
         "freshdesk": FreshdeskSource,
+        "trustpilot": TrustpilotSource,
         "phantombuster": PhantombusterSource,
         "elasticsearch": ElasticsearchSource,
         "attio": AttioSource,

{ingestr-0.13.55 → ingestr-0.13.56}/ingestr/src/sources.py

@@ -1362,17 +1362,25 @@ class S3Source:
             secret=secret_access_key[0],
         )
 
-        file_extension = path_to_file.split(".")[-1]
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
+        endpoint: Optional[str] = None
+        if "#" in table:
+            _, endpoint = table.split("#")
+            if endpoint not in ["csv", "jsonl", "parquet"]:
+                raise ValueError(
+                    "S3 Source only supports specific formats files: csv, jsonl, parquet"
+                )
+            endpoint = f"read_{endpoint}"
         else:
-            raise ValueError(
-                "S3 Source only supports specific formats files: csv, jsonl, parquet"
-            )
+            try:
+                endpoint = blob.parse_endpoint(path_to_file)
+            except blob.UnsupportedEndpointError:
+                raise ValueError(
+                    "S3 Source only supports specific formats files: csv, jsonl, parquet"
+                )
+            except Exception as e:
+                raise ValueError(
+                    f"Failed to parse endpoint from path: {path_to_file}"
+                ) from e
 
         from ingestr.src.filesystem import readers
 
@@ -1844,17 +1852,16 @@ class GCSSource:
             token=credentials,
         )
 
-        file_extension = path_to_file.split(".")[-1]
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
-        else:
+        try:
+            endpoint = blob.parse_endpoint(path_to_file)
+        except blob.UnsupportedEndpointError:
             raise ValueError(
-                "GCS Source only supports specific formats files: csv, jsonl, parquet"
+                "S3 Source only supports specific formats files: csv, jsonl, parquet"
             )
+        except Exception as e:
+            raise ValueError(
+                f"Failed to parse endpoint from path: {path_to_file}"
+            ) from e
 
         from ingestr.src.filesystem import readers
 
@@ -2392,6 +2399,47 @@ class FreshdeskSource:
         ).with_resources(table)
 
 
+class TrustpilotSource:
+    # trustpilot://<business_unit_id>?api_key=<api_key>
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        business_unit_id = parsed_uri.netloc
+        params = parse_qs(parsed_uri.query)
+
+        if not business_unit_id:
+            raise MissingValueError("business_unit_id", "Trustpilot")
+
+        api_key = params.get("api_key")
+        if api_key is None:
+            raise MissingValueError("api_key", "Trustpilot")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is None:
+            start_date = ensure_pendulum_datetime("2000-01-01").in_tz("UTC").isoformat()
+        else:
+            start_date = ensure_pendulum_datetime(start_date).in_tz("UTC").isoformat()
+
+        end_date = kwargs.get("interval_end")
+
+        if end_date is not None:
+            end_date = ensure_pendulum_datetime(end_date).in_tz("UTC").isoformat()
+
+        if table not in ["reviews"]:
+            raise UnsupportedResourceError(table, "Trustpilot")
+
+        from ingestr.src.trustpilot import trustpilot_source
+
+        return trustpilot_source(
+            business_unit_id=business_unit_id,
+            api_key=api_key[0],
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)
+
+
 class PhantombusterSource:
     def handles_incrementality(self) -> bool:
         return True
@@ -2622,18 +2670,15 @@ class SFTPSource:
         else:
             file_glob = f"/{table}"
 
-        file_extension = table.split(".")[-1].lower()
-        endpoint: str
-        if file_extension == "csv":
-            endpoint = "read_csv"
-        elif file_extension == "jsonl":
-            endpoint = "read_jsonl"
-        elif file_extension == "parquet":
-            endpoint = "read_parquet"
-        else:
+        try:
+            endpoint = blob.parse_endpoint(table)
+        except blob.UnsupportedEndpointError:
             raise ValueError(
-                "FTPServer Source only supports specific file formats: csv, jsonl, parquet."
+                "SFTP Source only supports specific formats files: csv, jsonl, parquet"
             )
+        except Exception as e:
+            raise ValueError(f"Failed to parse endpoint from path: {table}") from e
+
         from ingestr.src.filesystem import readers
 
         dlt_source_resource = readers(bucket_url, fs, file_glob)

ingestr-0.13.56/ingestr/src/trustpilot/__init__.py

@@ -0,0 +1,48 @@
+"""Trustpilot source for ingesting reviews."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .client import TrustpilotClient
+
+
+@dlt.source()
+def trustpilot_source(
+    business_unit_id: str,
+    start_date: str,
+    end_date: str | None,
+    api_key: str,
+    per_page: int = 1000,
+) -> Iterable[DltResource]:
+    """Return resources for Trustpilot."""
+
+    client = TrustpilotClient(api_key=api_key)
+
+    @dlt.resource(name="reviews", write_disposition="merge", primary_key="id")
+    def reviews(
+        dateTime=(
+            dlt.sources.incremental(
+                "updated_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Generator[Dict[str, Any], None, None]:
+        if end_date is None:
+            end_dt = pendulum.now(tz="UTC").isoformat()
+        else:
+            end_dt = dateTime.end_value
+        start_dt = dateTime.last_value
+        yield from client.paginated_reviews(
+            business_unit_id=business_unit_id,
+            per_page=per_page,
+            updated_since=start_dt,
+            end_date=end_dt,
+        )
+
+    yield reviews

ingestr-0.13.56/ingestr/src/trustpilot/client.py

@@ -0,0 +1,48 @@
+"""Simple Trustpilot API client."""
+
+from typing import Any, Dict, Iterable
+
+import pendulum
+from dlt.sources.helpers import requests
+
+
+class TrustpilotClient:
+    """Client for the Trustpilot public API."""
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+        self.base_url = "https://api.trustpilot.com/v1"
+
+    def _get(self, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        params = dict(params)
+        params["apikey"] = self.api_key
+        response = requests.get(f"{self.base_url}{endpoint}", params=params)
+        response.raise_for_status()
+        return response.json()
+
+    def paginated_reviews(
+        self,
+        business_unit_id: str,
+        updated_since: str,
+        end_date: str,
+        per_page: int = 1000,
+    ) -> Iterable[Dict[str, Any]]:
+        page = 1
+        while True:
+            params: Dict[str, Any] = {"perPage": per_page, "page": page}
+            if updated_since:
+                params["updatedSince"] = updated_since
+            data = self._get(f"/business-units/{business_unit_id}/reviews", params)
+            reviews = data.get("reviews", data)
+            if not reviews:
+                break
+            for review in reviews:
+                end_date_dt = pendulum.parse(end_date)
+                review["updated_at"] = review["updatedAt"]
+                review_dt = pendulum.parse(review["updated_at"])
+                if review_dt > end_date_dt:  # type: ignore
+                    continue
+                yield review
+            if len(reviews) < per_page:
+                break
+            page += 1

ingestr-0.13.55/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.55"