ingestr 0.12.8__tar.gz → 0.12.10__tar.gz

{ingestr-0.12.8 → ingestr-0.12.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.8
+Version: 0.12.10
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -22,6 +22,7 @@ Requires-Dist: dlt==1.5.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
+Requires-Dist: gcsfs==2024.10.0
 Requires-Dist: google-analytics-data==0.18.16
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0

{ingestr-0.12.8 → ingestr-0.12.10}/docs/.vitepress/config.mjs

@@ -102,12 +102,14 @@ export default defineConfig({
                 text: "Facebook Ads",
                 link: "/supported-sources/facebook-ads.md",
               },
+              { text: "Google Cloud Storage (GCS)", link: "/supported-sources/gcs.md" },
               { text: "Google Analytics", link: "/supported-sources/google_analytics.md" },
               { text: "GitHub", link: "/supported-sources/github.md" },
               { text: "Google Sheets", link: "/supported-sources/gsheets.md" },
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
+              { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },
               { text: "Notion", link: "/supported-sources/notion.md" },
               { text: "S3", link: "/supported-sources/s3.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },

ingestr-0.12.10/docs/supported-sources/gcs.md

@@ -0,0 +1,62 @@
+# Google Cloud Storage
+
+[Google Cloud Storage](https://cloud.google.com/storage?hl=en) is an online file storage web service for storing and accessing data on Google Cloud Platform infrastructure. The service combines the performance and scalability of Google's cloud with advanced security and sharing capabilities. It is an Infrastructure as a Service (IaaS), comparable to Amazon S3. 
+
+## URI format
+
+The URI format for Google Cloud Storage is as follows:
+
+```plaintext
+gs://<bucket_name>?credentials_path=/path/to/service-account.json>
+```
+
+URI parameters:
+
+- `bucket_name`: The name of the bucket
+- `credentials_path`: path to file containing your Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-account-overview)
+
+## Setting up a GCS Integration
+
+To use Google Cloud Storage source in `ingestr`, you will need:
+* A Google Cloud Project.
+* A Service Account with atleast [roles/storage.objectUser](https://cloud.google.com/storage/docs/access-control/iam-roles) IAM permission.
+* A Service Account key file for the corresponding service account.
+
+For more information on how to create a Service Account or it's keys, see [Create service accounts](https://cloud.google.com/iam/docs/service-accounts-create) and [Create or delete service account keys](https://cloud.google.com/iam/docs/keys-create-delete) on Google Cloud docs.
+
+## Example
+
+Let's assume that:
+* Service account key in available in the current directory, under the filename `service_account.json`. 
+* The bucket you want to load data from is called `my-org-bucket`
+* The source file is available at `/data/latest/dump.csv`
+* The data needs to be saved in a DuckDB database called `local.db`
+* The destination table name will be `public.latest_dump`
+
+You can run the following command line to achieve this:
+
+```sh
+ingestr ingest \
+    --source-uri "gs://my-org-bucket?credentials_path=$PWD/service_account.json" \
+    --source-table "/data/latest/dump.csv" \
+    --dest-uri "duckdb:///local.db" \
+    --dest-table "public.latest_dump"
+```
+
+## Supported File Formats
+`gs` source only supports loading files in the following formats:
+* `csv`: Comma Separated Values (supports Tab Separated Values as well)
+* `parquet`: [Apache Parquet](https://parquet.apache.org/) storage format.
+* `jsonl`: Line delimited JSON. see [https://jsonlines.org/](https://jsonlines.org/)
+
+## File Pattern
+`ingestr` supports [glob](https://en.wikipedia.org/wiki/Glob_(programming)) like pattern matching for `gs` source.
+This allows for a powerful pattern matching mechanism that allows you to specify multiple files in a single `--source-table`.
+
+Below are some examples of path patterns, each path pattern is a reference from the root of the bucket:
+
+- `**/*.csv`: Retrieves all the CSV files, regardless of how deep they are within the folder structure.
+- `*.csv`: Retrieves all the CSV files from the first level of a folder.
+- `myFolder/**/*.jsonl`: Retrieves all the JSONL files from anywhere under `myFolder`.
+- `myFolder/mySubFolder/users.parquet`: Retrieves the `users.parquet` file from `mySubFolder`.
+- `employees.jsonl`: Retrieves the `employees.jsonl` file from the root level of the bucket.

ingestr-0.12.10/docs/supported-sources/linkedin_ads.md

@@ -0,0 +1,109 @@
+# LinkedIn Ads
+LinkedIn Ads is a platform that allows businesses and marketers to create, manage, and analyze advertising campaigns.
+
+Ingestr supports LinkedIn Ads as a source.
+
+## URI format
+The URI format for LinkedIn Ads as a source is as follows:
+
+```plaintext
+linkedinads://?access_token=<access_token>&account_ids=<account_ids>
+```
+## URI parameters:
+- `access_token`(required): It is used for authentication and is necessary to access data and reports through the LinkedIn Ads API. The access token lets your app access data using the permissions you set in the Developer App for your LinkedIn account.
+- `account_ids`(required): The comma-separated list of Ad Account IDs specifies the LinkedIn Ad Accounts for which you want to retrieve data. These IDs uniquely identify the LinkedIn Ad Accounts associated with a company, business, or individual, depending on the ownership of the Ad Accounts. They are required to fetch data for campaigns, creatives, and other related resources.
+
+LinkedIn Ads requires an `access_token` and `account_ids` to retrieve reports from the [LinkedIn Ads API](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2024-11&tabs=http#analytics-finder). Please follow these steps to obtain the `access_token` and `account_ids`
+
+### Create a LinkedIn developer application to obtain an access token
+1. Log in to LinkedIn with a [developer account](https://www.linkedin.com/developers)
+2. Click Create App. Please fill out:
+    - App Name
+    - Your company's LinkedIn page
+    - Your company's privacy policy URL
+    - Your company logo
+- Accept the terms and click "Create App"
+3. To verify your app:
+   - Go to "Settings" tab
+   - Find "App Settings"
+   - Click "Verify" under Company
+   - Click "Generate URL"
+   - Send this URL to your Page Admin 
+   - Click "I'm done" and open the URL in a new tab to verify, if you are the admin of your page.
+   -  Go to the "Products" tab, and click "Request access" for the Advertising API. It will take about a few minutes to get approved and then you need to fill out the form where you have to provide your company name, website, and other details.
+
+#### Authorize your app and obtain access token
+1. Go to the "Auth" tab
+4. Click "OAuth 2.0 tools" which is on top right corner of the page [link](https://www.linkedin.com/developers/tools/oauth)
+5. Click "Create token"
+6. Choose these permissions:
+   - `r_ads`
+   - `r_ads_reporting`
+7. Click "Request access token"
+8. You will be redirected to an authorization page. Use your LinkedIn credentials to log in and authorize your app and obtain your Access Token and Refresh Token. Copy the Access Token.
+
+> [!NOTE]
+> Access tokens last for 2 months. After they expire, you'll need to make new ones using 
+> [LinkedIn's Token Generator](https://www.linkedin.com/developers/tools/oauth/token-generator).
+
+To find the Ad Account IDs, the ad account owner can refer to the detailed instructions provided in this [guide](https://www.linkedin.com/help/linkedin/answer/a424270/find-linkedin-ads-account-details?lang=en).
+
+## Table: Custom Reports    
+Custom reports allow you to retrieve data based on specific dimensions and metrics.
+
+Custom Table Format:
+```
+custom:<dimensions>:<metrics>
+```
+### Parameters:
+- `dimensions`(required): A comma-separated list of dimensions is required. It must include at least one of the following: `campaign`, `account`, or `creative`, along with one time-based dimension, either `date` or `month`.
+  - `date`: group the data in your report by day
+  - `month`: group the data in your report by month
+- `metrics`(required): A comma-separated list of [metrics](https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2024-11&tabs=http#metrics-available) to retrieve.
+
+> [!NOTE]
+> By default, ingestr fetches data from January 1, 2018 to today's date. You can specify a custom date range using the `--interval-start` and `--interval-end` parameters.
+
+### Example
+
+Retrieve data for campaign with `account_ids` id_123 and id_456:
+```sh
+ingestr ingest \                         
+    --source-uri "linkedinads://?access_token=token_123&account_ids=id_123,id_456" \
+    --source-table 'custom:campaign,date:impressions,clicks' \
+    --dest-uri 'duckdb:///linkedin.duckdb' \
+    --dest-table 'dest.campaign'
+```
+
+The applied parameters for the report are:
+- dimensions: `campaign`, `date`
+- metrics: `impressions`, `clicks`
+
+Retrieve data for creative with `account_ids` id_123 and id_456 for the date range from 2024-10-15 to 2024-12-31:
+```sh
+ingestr ingest \                         
+    --source-uri "linkedinads://?access_token=token_123&account_ids=id_123,id_456" \
+    --source-table 'custom:creative,month:impressions,shares,videoCompletions' \
+    --dest-uri 'duckdb:///linkedin.duckdb' \
+    --dest-table 'dest.creative'
+    --interval-start '2024-10-15'
+    --interval-end '2024-12-31'
+```
+The applied parameters for the report are:
+- dimensions: `creative`, `month`
+- metrics: `shares`, `impressions`, `videoCompletions`
+
+```sh
+ingestr ingest \                         
+    --source-uri "linkedinads://?access_token=token_123&account_ids=id_123,id_456" \
+    --source-table 'custom:account,month:totalEngagements,impressions,' \
+    --dest-uri 'duckdb:///linkedin.duckdb' \
+    --dest-table 'dest.account'
+```
+The applied parameters for the report are:
+- dimensions: `account`, `month`
+- metrics: `totalEngagements`, `impressions`
+
+This command will retrieve data and save it to the destination table in the DuckDB database.
+
+<img alt="linkedin_ads_img" src="../media/linkedin_ads.png"/>

{ingestr-0.12.8 → ingestr-0.12.10}/docs/supported-sources/s3.md

@@ -9,7 +9,7 @@ ingestr supports S3 as a source.
 The URI format for S3 is as follows:
 
 ```plaintext
-s3://<bucket_name>/<path_to_file>?access_key_id=<access_key_id>&secret_access_key=<secret_access_key>
+s3://<bucket_name>?access_key_id=<access_key_id>&secret_access_key=<secret_access_key>
 ```
 
 URI parameters:
@@ -25,7 +25,11 @@ S3 requires an `access_key_id` and a `secret_access_key` to access the bucket. P
 For example, if your `access_key_id` is `AKC3YOW7E`, `secret_access_key` is `XCtkpL5B`, bucket name is `my_bucket`, and `path_to_files` is `students/students_details.csv`, here's a sample command that will copy the data from the S3 bucket into a DuckDB database:
 
 ```sh
-ingestr ingest --source-uri 's3://my_bucket/students/students_details.csv?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' --source-table 'students_details' --dest-uri duckdb:///s3.duckdb --dest-table 'dest.students_details'
+ingestr ingest \
+    --source-uri 's3://my_bucket?access_key_id=AKC3YOW7E&secret_access_key=XCtkpL5B' \
+    --source-table '/students/students_details.csv' \
+    --dest-uri duckdb:///s3.duckdb \
+    --dest-table 'dest.students_details'
 ```
 
 The result of this command will be a table in the DuckDB database in the path `s3.duckdb`.

{ingestr-0.12.8 → ingestr-0.12.10}/ingestr/src/appstore/__init__.py

@@ -28,6 +28,10 @@ def app_store(
     start_date: Optional[datetime] = None,
     end_date: Optional[datetime] = None,
 ) -> Iterable[DltResource]:
+    if start_date and start_date.tzinfo is not None:
+        start_date = start_date.replace(tzinfo=None)
+    if end_date and end_date.tzinfo is not None:
+        end_date = end_date.replace(tzinfo=None)
     for resource in RESOURCES:
         yield dlt.resource(
             get_analytics_reports,

{ingestr-0.12.8 → ingestr-0.12.10}/ingestr/src/factory.py

@@ -25,6 +25,7 @@ from ingestr.src.sources import (
     ChessSource,
     DynamoDBSource,
     FacebookAdsSource,
+    GCSSource,
     GitHubSource,
     GoogleAnalyticsSource,
     GoogleSheetsSource,
@@ -32,6 +33,7 @@ from ingestr.src.sources import (
     HubspotSource,
     KafkaSource,
     KlaviyoSource,
+    LinkedInAdsSource,
     LocalCsvSource,
     MongoDbSource,
     NotionSource,
@@ -124,6 +126,8 @@ class SourceDestinationFactory:
         "tiktok": TikTokSource,
         "googleanalytics": GoogleAnalyticsSource,
         "appstore": AppleAppStoreSource,
+        "gs": GCSSource,
+        "linkedinads": LinkedInAdsSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

{ingestr-0.12.8 → ingestr-0.12.10}/ingestr/src/filesystem/__init__.py

@@ -39,8 +39,6 @@ def readers(
     filesystem_resource = filesystem(bucket_url, credentials, file_glob=file_glob)
     filesystem_resource.apply_hints(
         incremental=dlt.sources.incremental("modification_date"),
-        range_end="closed",
-        range_start="closed",
     )
     return (
         filesystem_resource | dlt.transformer(name="read_csv")(_read_csv),

ingestr-0.12.10/ingestr/src/linkedin_ads/__init__.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+from .helpers import LinkedInAdsAPI, find_intervals
+
+
+@dlt.source(max_table_nesting=0)
+def linked_in_ads_source(
+    start_date: Date,
+    end_date: Date | None,
+    access_token: str,
+    account_ids: list[str],
+    dimension: Dimension,
+    metrics: list[str],
+    time_granularity: TimeGranularity,
+) -> DltResource:
+    linkedin_api = LinkedInAdsAPI(
+        access_token=access_token,
+        account_ids=account_ids,
+        dimension=dimension,
+        metrics=metrics,
+        time_granularity=time_granularity,
+    )
+
+    if time_granularity == TimeGranularity.daily:
+        primary_key = [dimension.value, "date"]
+        incremental_loading_param = "date"
+    else:
+        primary_key = [dimension.value, "start_date", "end_date"]
+        incremental_loading_param = "start_date"
+
+    @dlt.resource(write_disposition="merge", primary_key=primary_key)
+    def custom_reports(
+        dateTime=(
+            dlt.sources.incremental(
+                incremental_loading_param,
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_date = pendulum.now().date()
+        else:
+            end_date = dateTime.end_value
+
+        list_of_interval = find_intervals(
+            start_date=dateTime.last_value,
+            end_date=end_date,
+            time_granularity=time_granularity,
+        )
+        for start, end in list_of_interval:
+            yield linkedin_api.fetch_pages(start, end)
+
+    return custom_reports

ingestr-0.12.10/ingestr/src/linkedin_ads/dimension_time_enum.py

@@ -0,0 +1,12 @@
+from enum import Enum
+
+
+class Dimension(Enum):
+    campaign = "campaign"
+    creative = "creative"
+    account = "account"
+
+
+class TimeGranularity(Enum):
+    daily = "DAILY"
+    monthly = "MONTHLY"

ingestr-0.12.10/ingestr/src/linkedin_ads/helpers.py

@@ -0,0 +1,148 @@
+from urllib.parse import quote
+
+import pendulum
+import requests
+from dlt.sources.helpers.requests import Client
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        request_timeout=10.0,
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def flat_structure(items, pivot: Dimension, time_granularity: TimeGranularity):
+    for item in items:
+        if "pivotValues" in item:
+            if len(item["pivotValues"]) > 1:
+                item[pivot.value.lower()] = item["pivotValues"]
+            else:
+                item[pivot.value.lower()] = item["pivotValues"][0]
+        if "dateRange" in item:
+            start_date = item["dateRange"]["start"]
+            start_dt = pendulum.date(
+                year=start_date["year"],
+                month=start_date["month"],
+                day=start_date["day"],
+            )
+            if time_granularity == TimeGranularity.daily:
+                item["date"] = start_dt
+            else:
+                end_date = item["dateRange"]["end"]
+                end_dt = pendulum.date(
+                    year=end_date["year"],
+                    month=end_date["month"],
+                    day=end_date["day"],
+                )
+                item["start_date"] = start_dt
+                item["end_date"] = end_dt
+
+        del item["dateRange"]
+        del item["pivotValues"]
+
+    return items
+
+
+def find_intervals(start_date: Date, end_date: Date, time_granularity: TimeGranularity):
+    intervals = []
+
+    if start_date > end_date:
+        raise ValueError("Start date must be less than end date")
+
+    while start_date <= end_date:
+        if time_granularity == TimeGranularity.daily:
+            next_date = min(start_date.add(months=6), end_date)
+        else:
+            next_date = min(start_date.add(years=2), end_date)
+
+        intervals.append((start_date, next_date))
+
+        start_date = next_date.add(days=1)
+
+    return intervals
+
+
+def construct_url(
+    start: Date,
+    end: Date,
+    account_ids: list[str],
+    metrics: list[str],
+    dimension: Dimension,
+    time_granularity: TimeGranularity,
+):
+    date_range = f"(start:(year:{start.year},month:{start.month},day:{start.day})"
+    date_range += f",end:(year:{end.year},month:{end.month},day:{end.day}))"
+    accounts = ",".join(
+        [quote(f"urn:li:sponsoredAccount:{account_id}") for account_id in account_ids]
+    )
+    encoded_accounts = f"List({accounts})"
+    dimension_str = dimension.value.upper()
+    time_granularity_str = time_granularity.value
+    metrics_str = ",".join([metric for metric in metrics])
+
+    url = (
+        f"https://api.linkedin.com/rest/adAnalytics?"
+        f"q=analytics&timeGranularity={time_granularity_str}&"
+        f"dateRange={date_range}&accounts={encoded_accounts}&"
+        f"pivot={dimension_str}&fields={metrics_str}"
+    )
+
+    return url
+
+
+class LinkedInAdsAPI:
+    def __init__(
+        self,
+        access_token,
+        time_granularity,
+        account_ids,
+        dimension,
+        metrics,
+    ):
+        self.time_granularity: TimeGranularity = time_granularity
+        self.account_ids: list[str] = account_ids
+        self.dimension: Dimension = dimension
+        self.metrics: list[str] = metrics
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Linkedin-Version": "202411",
+            "X-Restli-Protocol-Version": "2.0.0",
+        }
+
+    def fetch_pages(self, start: Date, end: Date):
+        client = create_client()
+        url = construct_url(
+            start=start,
+            end=end,
+            account_ids=self.account_ids,
+            metrics=self.metrics,
+            dimension=self.dimension,
+            time_granularity=self.time_granularity,
+        )
+        response = client.get(url=url, headers=self.headers)
+
+        if response.status_code != 200:
+            error_data = response.json()
+            raise ValueError(f"LinkedIn API Error: {error_data.get('message')}")
+
+        result = response.json()
+        items = result.get("elements", [])
+        yield flat_structure(
+            items=items,
+            pivot=self.dimension,
+            time_granularity=self.time_granularity,
+        )