omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/stripe_analytics/settings.py

@@ -0,0 +1,80 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Stripe analytics source settings and constants"""
+
+# the most popular endpoints
+# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
+ENDPOINTS = {
+    "account": "Account",
+    "applepaydomain": "ApplePayDomain",
+    "apple_pay_domain": "ApplePayDomain",
+    "applicationfee": "ApplicationFee",
+    "application_fee": "ApplicationFee",
+    "checkoutsession": "CheckoutSession",
+    "checkout_session": "CheckoutSession",
+    "coupon": "Coupon",
+    "charge": "Charge",
+    "customer": "Customer",
+    "dispute": "Dispute",
+    "paymentintent": "PaymentIntent",
+    "payment_intent": "PaymentIntent",
+    "paymentlink": "PaymentLink",
+    "payment_link": "PaymentLink",
+    "paymentmethod": "PaymentMethod",
+    "payment_method": "PaymentMethod",
+    "paymentmethoddomain": "PaymentMethodDomain",
+    "payment_method_domain": "PaymentMethodDomain",
+    "payout": "Payout",
+    "plan": "Plan",
+    "price": "Price",
+    "product": "Product",
+    "promotioncode": "PromotionCode",
+    "promotion_code": "PromotionCode",
+    "quote": "Quote",
+    "refund": "Refund",
+    "review": "Review",
+    "setupattempt": "SetupAttempt",
+    "setup_attempt": "SetupAttempt",
+    "setupintent": "SetupIntent",
+    "setup_intent": "SetupIntent",
+    "shippingrate": "ShippingRate",
+    "shipping_rate": "ShippingRate",
+    "subscription": "Subscription",
+    "subscriptionitem": "SubscriptionItem",
+    "subscription_item": "SubscriptionItem",
+    "subscriptionschedule": "SubscriptionSchedule",
+    "subscription_schedule": "SubscriptionSchedule",
+    "transfer": "Transfer",
+    "taxcode": "TaxCode",
+    "tax_code": "TaxCode",
+    "taxid": "TaxId",
+    "tax_id": "TaxId",
+    "taxrate": "TaxRate",
+    "tax_rate": "TaxRate",
+    "topup": "Topup",
+    "top_up": "Topup",
+    "webhookendpoint": "WebhookEndpoint",
+    "webhook_endpoint": "WebhookEndpoint",
+    "invoice": "Invoice",
+    "invoiceitem": "InvoiceItem",
+    "invoice_item": "InvoiceItem",
+    "invoicelineitem": "InvoiceLineItem",
+    "invoice_line_item": "InvoiceLineItem",
+    "balancetransaction": "BalanceTransaction",
+    "balance_transaction": "BalanceTransaction",
+    "creditnote": "CreditNote",
+    "credit_note": "CreditNote",
+    "event": "Event",
+}

omniload/src/table_definition.py

@@ -0,0 +1,15 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class TableDefinition:
+    dataset: str
+    table: str
+
+
+def table_string_to_dataclass(table: str) -> TableDefinition:
+    table_fields = table.split(".", 1)
+    if len(table_fields) != 2:
+        raise ValueError("Table name must be in the format <schema>.<table>")
+
+    return TableDefinition(dataset=table_fields[0], table=table_fields[1])

omniload/src/testdata/fakebqcredentials.json

@@ -0,0 +1,14 @@
+{
+    "type": "service_account",
+    "project_id": "my-project",
+    "private_key_id": "some-private-key-id",
+    "private_key": "-----BEGIN PRIVATE KEY-----some-private-key-----END PRIVATE KEY-----\n",
+    "client_email": "someuser@someproject.iam.gserviceaccount.com",
+    "client_id": "1234567890",
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/",
+    "universe_domain": "googleapis.com"
+  }
+  

omniload/src/tiktok_ads/__init__.py

@@ -0,0 +1,150 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .tiktok_helpers import TikTokAPI
+
+KNOWN_TYPE_HINTS = {
+    "spend": {"data_type": "decimal"},
+    "billed_cost": {"data_type": "decimal"},
+    "cash_spend": {"data_type": "decimal"},
+    "voucher_spend": {"data_type": "decimal"},
+    "cpc": {"data_type": "decimal"},
+    "cpm": {"data_type": "decimal"},
+    "impressions": {"data_type": "bigint"},
+    "gross_impressions": {"data_type": "bigint"},
+    "clicks": {"data_type": "bigint"},
+    "ctr": {"data_type": "decimal"},
+    "reach": {"data_type": "bigint"},
+    "cost_per_1000_reached": {"data_type": "decimal"},
+    "frequency": {"data_type": "decimal"},
+    "conversion": {"data_type": "bigint"},
+    "cost_per_conversion": {"data_type": "decimal"},
+    "conversion_rate": {"data_type": "decimal"},
+    "conversion_rate_v2": {"data_type": "decimal"},
+    "real_time_conversion": {"data_type": "bigint"},
+    "real_time_cost_per_conversion": {"data_type": "decimal"},
+    "real_time_conversion_rate": {"data_type": "decimal"},
+    "real_time_conversion_rate_v2": {"data_type": "decimal"},
+    "result": {"data_type": "bigint"},
+    "cost_per_result": {"data_type": "decimal"},
+    "result_rate": {"data_type": "decimal"},
+    "real_time_result": {"data_type": "bigint"},
+    "real_time_cost_per_result": {"data_type": "decimal"},
+    "real_time_result_rate": {"data_type": "decimal"},
+    "secondary_goal_result": {"data_type": "bigint"},
+    "cost_per_secondary_goal_result": {"data_type": "decimal"},
+    "secondary_goal_result_rate": {"data_type": "decimal"},
+}
+
+
+def find_intervals(
+    current_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
+    interval_days: int,
+):
+    intervals = []
+    while current_date <= end_date:
+        interval_end = min(current_date.add(days=interval_days), end_date)
+        intervals.append((current_date, interval_end))
+        current_date = interval_end.add(days=1)
+
+    return intervals
+
+
+@dlt.source(max_table_nesting=0)
+def tiktok_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime,
+    access_token: str,
+    advertiser_ids: list[str],
+    timezone: str,
+    page_size: int,
+    filtering_param: bool,
+    filter_name: str,
+    filter_value: list[int],
+    dimensions: list[str],
+    metrics: list[str],
+) -> DltResource:
+    tiktok_api = TikTokAPI(
+        access_token=access_token,
+        timezone=timezone,
+        page_size=page_size,
+        filtering_param=filtering_param,
+        filter_name=filter_name,
+        filter_value=filter_value,
+    )
+    incremental_loading_param = ""
+    is_incremental = False
+    interval_days = 365
+
+    if "stat_time_day" in dimensions:
+        incremental_loading_param = "stat_time_day"
+        is_incremental = True
+        interval_days = 30
+
+    if "stat_time_hour" in dimensions:
+        incremental_loading_param = "stat_time_hour"
+        is_incremental = True
+        interval_days = 0
+
+    type_hints = {
+        "advertiser_id": {"data_type": "text"},
+    }
+    for dimension in dimensions:
+        if dimension in KNOWN_TYPE_HINTS:
+            type_hints[dimension] = KNOWN_TYPE_HINTS[dimension]
+    for metric in metrics:
+        if metric in KNOWN_TYPE_HINTS:
+            type_hints[metric] = KNOWN_TYPE_HINTS[metric]
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key=dimensions + ["advertiser_id"],
+        columns=type_hints,
+        parallelized=True,
+    )
+    def custom_reports(
+        datetime=(
+            dlt.sources.incremental(
+                incremental_loading_param,
+                initial_value=start_date,
+                end_value=end_date,
+                range_end="closed",
+                range_start="closed",
+            )
+            if is_incremental
+            else None
+        ),
+    ) -> Iterable[TDataItem]:
+        start_date_tz_adjusted = start_date.in_tz(timezone)
+        end_date_tz_adjusted = end_date.in_tz(timezone)
+
+        if datetime is not None:
+            start_date_tz_adjusted = ensure_pendulum_datetime(
+                datetime.last_value
+            ).in_tz(timezone)
+            end_date_tz_adjusted = ensure_pendulum_datetime(datetime.end_value).in_tz(
+                timezone
+            )
+
+        list_of_interval = find_intervals(
+            current_date=start_date_tz_adjusted,
+            end_date=end_date_tz_adjusted,
+            interval_days=interval_days,
+        )
+
+        for start, end in list_of_interval:
+            yield tiktok_api.fetch_pages(
+                advertiser_ids=advertiser_ids,
+                start_time=start,
+                end_time=end,
+                dimensions=dimensions,
+                metrics=metrics,
+            )
+
+    return custom_reports

omniload/src/tiktok_ads/tiktok_helpers.py

@@ -0,0 +1,130 @@
+import json
+
+import requests
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources.helpers.requests import Client
+
+BASE_URL = "https://business-api.tiktok.com/open_api/v1.3/report/integrated/get/"
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+def flat_structure(items, timezone="UTC"):
+    for item in items:
+        if "dimensions" in item:
+            for key, value in item["dimensions"].items():
+                if key == "stat_time_day":
+                    item["stat_time_day"] = ensure_pendulum_datetime(value).in_tz(
+                        timezone
+                    )
+                elif key == "stat_time_hour":
+                    item["stat_time_hour"] = ensure_pendulum_datetime(value).in_tz(
+                        timezone
+                    )
+                else:
+                    item[key] = value
+            del item["dimensions"]
+
+            for key, value in item["metrics"].items():
+                item[key] = value
+            del item["metrics"]
+
+    return items
+
+
+class TikTokAPI:
+    def __init__(
+        self,
+        access_token,
+        timezone,
+        page_size,
+        filtering_param,
+        filter_name,
+        filter_value,
+    ):
+        self.headers = {
+            "Access-Token": access_token,
+        }
+        self.timezone = timezone
+        self.page_size = page_size
+        self.filtering_param = filtering_param
+        self.filter_name = filter_name
+        self.filter_value = filter_value
+
+    def fetch_pages(
+        self, advertiser_ids: list[str], start_time, end_time, dimensions, metrics
+    ):
+        data_level_mapping = {
+            "advertiser_id": "AUCTION_ADVERTISER",
+            "campaign_id": "AUCTION_CAMPAIGN",
+            "adgroup_id": "AUCTION_ADGROUP",
+        }
+
+        data_level = "AUCTION_AD"
+        for id_dimension in dimensions:
+            if id_dimension in data_level_mapping:
+                data_level = data_level_mapping[id_dimension]
+                break
+
+        current_page = 1
+        start_time = ensure_pendulum_datetime(start_time).to_date_string()
+        end_time = ensure_pendulum_datetime(end_time).to_date_string()
+
+        filtering = [
+            {
+                "field_name": self.filter_name,
+                "filter_type": "IN",
+                "filter_value": json.dumps(self.filter_value),
+            }
+        ]
+        params = {
+            "advertiser_ids": json.dumps(advertiser_ids),
+            "report_type": "BASIC",
+            "data_level": data_level,
+            "start_date": start_time,
+            "end_date": end_time,
+            "page_size": self.page_size,
+            "dimensions": json.dumps(dimensions),
+            "metrics": json.dumps(metrics),
+        }
+
+        if self.filtering_param:
+            params["filtering"] = json.dumps(filtering)
+        client = create_client()
+        while True:
+            params["page"] = current_page
+            response = client.get(url=BASE_URL, headers=self.headers, params=params)
+
+            result = response.json()
+            if result.get("message") != "OK":
+                raise ValueError(result.get("message", ""))
+
+            result_data = result.get("data", {})
+            items = result_data.get("list", [])
+
+            flat_structure(items=items, timezone=self.timezone)
+
+            yield items
+
+            page_info = result_data.get("page_info", {})
+            total_pages = page_info.get("total_page", 1)
+
+            if current_page >= total_pages:
+                break
+
+            current_page += 1

omniload/src/time.py

@@ -0,0 +1,11 @@
+import datetime
+from typing import Optional
+
+
+def isotime(dt: Optional[datetime.datetime]) -> Optional[str]:
+    """
+    Converts a datetime object to an iso 8601 formatted string.
+    """
+    if dt is None:
+        return None
+    return dt.isoformat()

omniload/src/trustpilot/__init__.py

@@ -0,0 +1,48 @@
+"""Trustpilot source for ingesting reviews."""
+
+from typing import Any, Dict, Generator, Iterable
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .client import TrustpilotClient
+
+
+@dlt.source()
+def trustpilot_source(
+    business_unit_id: str,
+    start_date: str,
+    end_date: str | None,
+    api_key: str,
+    per_page: int = 1000,
+) -> Iterable[DltResource]:
+    """Return resources for Trustpilot."""
+
+    client = TrustpilotClient(api_key=api_key)
+
+    @dlt.resource(name="reviews", write_disposition="merge", primary_key="id")
+    def reviews(
+        dateTime=(
+            dlt.sources.incremental(
+                "updated_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Generator[Dict[str, Any], None, None]:
+        if end_date is None:
+            end_dt = pendulum.now(tz="UTC").isoformat()
+        else:
+            end_dt = dateTime.end_value
+        start_dt = dateTime.last_value
+        yield from client.paginated_reviews(
+            business_unit_id=business_unit_id,
+            per_page=per_page,
+            updated_since=start_dt,
+            end_date=end_dt,
+        )
+
+    yield reviews

omniload/src/trustpilot/client.py

@@ -0,0 +1,48 @@
+"""Simple Trustpilot API client."""
+
+from typing import Any, Dict, Iterable
+
+import pendulum
+from dlt.sources.helpers import requests
+
+
+class TrustpilotClient:
+    """Client for the Trustpilot public API."""
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+        self.base_url = "https://api.trustpilot.com/v1"
+
+    def _get(self, endpoint: str, params: Dict[str, Any]) -> Dict[str, Any]:
+        params = dict(params)
+        params["apikey"] = self.api_key
+        response = requests.get(f"{self.base_url}{endpoint}", params=params)
+        response.raise_for_status()
+        return response.json()
+
+    def paginated_reviews(
+        self,
+        business_unit_id: str,
+        updated_since: str,
+        end_date: str,
+        per_page: int = 1000,
+    ) -> Iterable[Dict[str, Any]]:
+        page = 1
+        while True:
+            params: Dict[str, Any] = {"perPage": per_page, "page": page}
+            if updated_since:
+                params["updatedSince"] = updated_since
+            data = self._get(f"/business-units/{business_unit_id}/reviews", params)
+            reviews = data.get("reviews", data)
+            if not reviews:
+                break
+            for review in reviews:
+                end_date_dt = pendulum.parse(end_date)
+                review["updated_at"] = review["updatedAt"]
+                review_dt = pendulum.parse(review["updated_at"])
+                if review_dt > end_date_dt:  # type: ignore
+                    continue
+                yield review
+            if len(reviews) < per_page:
+                break
+            page += 1

omniload/src/version.py

@@ -0,0 +1,6 @@
+try:
+    from omniload.src import buildinfo  # type: ignore[import-not-found,attr-defined]
+
+    __version__ = buildinfo.version.lstrip("v")
+except ImportError:
+    __version__ = "0.0.0-dev"

omniload/src/wise/__init__.py

@@ -0,0 +1,68 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+
+from .client import WiseClient
+
+
+@dlt.source(max_table_nesting=0)
+def wise_source(
+    api_key: str,
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    client = WiseClient(api_key)
+
+    # List of all profiles belonging to user.
+    @dlt.resource(write_disposition="merge", name="profiles", primary_key="id")
+    def profiles() -> Iterable[TDataItem]:
+        yield from client.fetch_profiles()
+
+    # List transfers for a profile.
+    @dlt.resource(write_disposition="merge", name="transfers", primary_key="id")
+    def transfers(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "created",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ):
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_transfers(profile["id"], start_dt, end_dt)
+
+    # Retrieve the user's multi-currency account balance accounts. It returns all balance accounts the profile has.
+    @dlt.resource(write_disposition="merge", name="balances", primary_key="id")
+    def balances(
+        profiles=profiles,
+        datetime=dlt.sources.incremental(
+            "modificationTime",
+            initial_value=start_date,
+            end_value=end_date,
+            range_end="closed",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        if datetime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = datetime.end_value
+
+        start_dt = datetime.last_value
+
+        for profile in profiles:
+            yield from client.fetch_balances(profile["id"], start_dt, end_dt)
+
+    return profiles, transfers, balances

omniload/src/wise/client.py

@@ -0,0 +1,63 @@
+from typing import Iterable
+
+import pendulum
+from dlt.sources.helpers.requests import Client
+
+
+class WiseClient:
+    BASE_URL = "https://api.transferwise.com"
+
+    def __init__(self, api_key: str) -> None:
+        self.session = Client(raise_for_status=False).session
+        self.session.headers.update({"Authorization": f"Bearer {api_key}"})
+
+    # https://docs.wise.com/api-docs/api-reference/profile#list-profiles
+    def fetch_profiles(self) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v2/profiles"
+        resp = self.session.get(url)
+        resp.raise_for_status()
+        for profile in resp.json():
+            yield profile
+
+    # https://docs.wise.com/api-docs/api-reference/transfer#list-transfers
+    def fetch_transfers(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ):
+        offset = 0
+
+        while True:
+            data = self.session.get(
+                f"{self.BASE_URL}/v1/transfers",
+                params={
+                    "profile": profile_id,
+                    "createdDateStart": start_time.to_date_string(),
+                    "createdDateEnd": end_time.to_date_string(),
+                    "limit": 100,
+                    "offset": offset,
+                },
+            )
+            response_data = data.json()
+
+            if not response_data or len(response_data) == 0:
+                break
+
+            for transfer in response_data:
+                transfer["created"] = pendulum.parse(transfer["created"])
+
+                yield transfer
+            offset += 100
+
+    # https://docs.wise.com/api-docs/api-reference/balance#list
+    def fetch_balances(
+        self, profile_id: str, start_time=pendulum.DateTime, end_time=pendulum.DateTime
+    ) -> Iterable[dict]:
+        url = f"{self.BASE_URL}/v4/profiles/{profile_id}/balances"
+        resp = self.session.get(url, params={"types": "STANDARD,SAVINGS"})
+        resp.raise_for_status()
+        for balance in resp.json():
+            balance["modificationTime"] = pendulum.parse(balance["modificationTime"])
+            if (
+                balance["modificationTime"] > start_time
+                and balance["modificationTime"] < end_time
+            ):
+                yield balance