omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/snapchat_ads/settings.py

@@ -0,0 +1,130 @@
+"""Snapchat Ads source settings and constants"""
+
+from typing import Literal
+
+from dlt.common.schema.typing import TColumnSchema
+
+# Valid granularities for stats (required)
+TStatsGranularity = Literal["TOTAL", "DAY", "HOUR", "LIFETIME"]
+
+# Valid breakdowns for stats (object-level breakdown)
+TStatsBreakdown = Literal["ad", "adsquad", "campaign"]
+
+# Valid dimensions for stats (insight-level breakdown)
+TStatsDimension = Literal["GEO", "DEMO", "INTEREST", "DEVICE"]
+
+# Valid pivots for stats (pivot for insights breakdown)
+TStatsPivot = Literal[
+    "country",
+    "region",
+    "dma",
+    "gender",
+    "age_bucket",
+    "interest_category_id",
+    "interest_category_name",
+    "operating_system",
+    "make",
+    "model",
+]
+
+# Sets for efficient lookup
+VALID_GRANULARITIES = {"TOTAL", "DAY", "HOUR", "LIFETIME"}
+VALID_BREAKDOWNS = {"ad", "adsquad", "campaign"}
+VALID_DIMENSIONS = {"GEO", "DEMO", "INTEREST", "DEVICE"}
+VALID_PIVOTS = {
+    "country",
+    "region",
+    "dma",
+    "gender",
+    "age_bucket",
+    "interest_category_id",
+    "interest_category_name",
+    "operating_system",
+    "make",
+    "model",
+}
+
+# Stats primary key - includes all possible identifying fields
+# campaign_id is always present
+# adsquad_id and ad_id will be NULL when no breakdown is specified
+# Time fields identify the time period for the stats
+STATS_PRIMARY_KEY = [
+    "campaign_id",
+    "adsquad_id",
+    "ad_id",
+    "start_time",
+    "end_time",
+]
+
+# Default stats fields
+DEFAULT_STATS_FIELDS = "impressions,spend"
+
+# Core metrics column hints for stats resources
+# All monetary values are in micro-currency (1.00 = 1,000,000 micro-currency)
+# Metrics are finalized 48 hours after the end of the day in the Ad Account's timezone
+STATS_METRICS_COLUMNS: dict[str, TColumnSchema] = {
+    # Core metrics (available for all Snap Ads)
+    "impressions": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Impression Count",
+    },
+    "swipes": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Swipe-Up Count",
+    },
+    "view_time_millis": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Deprecated: Use screen_time_millis instead. Total Time Spent on top Snap Ad (milliseconds)",
+    },
+    "screen_time_millis": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Total Time Spent on top Snap Ad (milliseconds)",
+    },
+    "quartile_1": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Video Views to 25%",
+    },
+    "quartile_2": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Video Views to 50%",
+    },
+    "quartile_3": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Video Views to 75%",
+    },
+    "view_completion": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Video Views to completion",
+    },
+    "spend": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Amount Spent (micro-currency: 1.00 = 1,000,000)",
+    },
+    "coupon_used_local": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Amount Spent via Coupon in the assigned currency of Ad Account (micro-currency)",
+    },
+    "coupon_used_usd": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Amount Spent via Coupon in USD (micro-currency)",
+    },
+    "video_views": {
+        "data_type": "bigint",
+        "nullable": True,
+        "description": "Total impressions with at least 2 seconds of consecutive watch time or a swipe up action",
+    },
+}
+
+# Set of valid metric field names for validation
+VALID_METRICS = set(STATS_METRICS_COLUMNS.keys())

omniload/src/socrata_source/__init__.py

@@ -0,0 +1,83 @@
+"""A source loading data from Socrata open data platform"""
+
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+
+from .helpers import fetch_data
+
+
+@dlt.source(name="socrata", max_table_nesting=0)
+def source(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental: Optional[Any] = None,
+    primary_key: Optional[str] = None,
+    write_disposition: Optional[str] = dlt.config.value,
+):
+    """
+    A dlt source for the Socrata open data platform.
+
+    Supports both full refresh (replace) and incremental loading (merge).
+
+    Args:
+        domain: The Socrata domain (e.g., "evergreen.data.socrata.com")
+        dataset_id: The dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits (recommended)
+        username: Username for authentication (if dataset is private)
+        password: Password for authentication (if dataset is private)
+        incremental: DLT incremental object for incremental loading
+        primary_key: Primary key field for merge operations (default: ":id")
+        write_disposition: Write disposition ("replace", "append", "merge").
+            If not provided, automatically determined based on incremental setting.
+
+    Returns:
+        A dlt source with a single "dataset" resource
+    """
+
+    @dlt.resource(
+        write_disposition=write_disposition or "replace",
+        primary_key=primary_key,  # type: ignore[call-overload]
+    )
+    def dataset(
+        incremental: Optional[dlt.sources.incremental] = incremental,  # type: ignore[type-arg]
+    ) -> Iterator[Dict[str, Any]]:
+        """
+        Yields records from a Socrata dataset.
+
+        Supports both full refresh (replace) and incremental loading (merge).
+        When incremental is provided, filters data using SoQL WHERE clause on the server side.
+
+        Yields:
+            Dict[str, Any]: Individual records from the dataset
+        """
+        fetch_kwargs: Dict[str, Any] = {
+            "domain": domain,
+            "dataset_id": dataset_id,
+            "app_token": app_token,
+            "username": username,
+            "password": password,
+        }
+
+        if incremental and incremental.cursor_path:
+            fetch_kwargs["incremental_key"] = incremental.cursor_path
+            fetch_kwargs["start_value"] = (
+                str(incremental.last_value)
+                if incremental.last_value is not None
+                else None
+            )
+            if getattr(incremental, "end_value", None) is not None:
+                ev = incremental.end_value  # type: ignore[attr-defined]
+                fetch_kwargs["end_value"] = (
+                    ev.isoformat()  # type: ignore[union-attr]
+                    if hasattr(ev, "isoformat")
+                    else str(ev)
+                )
+
+        # Fetch and yield records
+        yield from fetch_data(**fetch_kwargs)
+
+    return (dataset,)

omniload/src/socrata_source/helpers.py

@@ -0,0 +1,85 @@
+"""Socrata API helpers"""
+
+from typing import Any, Dict, Iterator, Optional
+
+from dlt.sources.helpers import requests
+
+from .settings import DEFAULT_PAGE_SIZE, REQUEST_TIMEOUT
+
+
+def fetch_data(
+    domain: str,
+    dataset_id: str,
+    app_token: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    incremental_key: Optional[str] = None,
+    start_value: Optional[str] = None,
+    end_value: Optional[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Fetch records from Socrata dataset with pagination and optional filtering.
+
+    Uses offset-based pagination to get all records, not just first 50000.
+    Supports incremental loading via SoQL WHERE clause for server-side filtering.
+
+    Args:
+        domain: Socrata domain (e.g., "data.seattle.gov")
+        dataset_id: Dataset identifier (e.g., "6udu-fhnu")
+        app_token: Socrata app token for higher rate limits
+        username: Username for authentication
+        password: Password for authentication
+        start_value: Minimum value for incremental_key (inclusive)
+        end_value: Maximum value for incremental_key (exclusive)
+
+    Yields:
+        Lists of records (one list per page)
+
+    Raises:
+        requests.HTTPError: If API request fails
+    """
+    url = f"https://{domain}/resource/{dataset_id}.json"
+
+    headers = {"Accept": "application/json"}
+    if app_token:
+        headers["X-App-Token"] = app_token
+
+    auth = (username, password) if username and password else None
+
+    limit = DEFAULT_PAGE_SIZE
+    offset = 0
+
+    while True:
+        params: Dict[str, Any] = {"$limit": limit, "$offset": offset}
+
+        if incremental_key and start_value:
+            start_value_iso = str(start_value).replace(" ", "T")
+            where_conditions = [f"{incremental_key} >= '{start_value_iso}'"]
+
+            if end_value:
+                end_value_iso = str(end_value).replace(" ", "T")
+                where_conditions.append(f"{incremental_key} < '{end_value_iso}'")
+
+            params["$where"] = " AND ".join(where_conditions)
+            params["$order"] = f"{incremental_key} ASC"
+
+        response = requests.get(
+            url,
+            headers=headers,
+            auth=auth,
+            params=params,
+            timeout=REQUEST_TIMEOUT,
+        )
+        response.raise_for_status()
+
+        data = response.json()
+
+        if not data:
+            break
+
+        yield data
+
+        if len(data) < limit:
+            break
+
+        offset += limit

omniload/src/socrata_source/settings.py

@@ -0,0 +1,8 @@
+"""Socrata API settings and constants"""
+
+# Request timeout in seconds
+REQUEST_TIMEOUT = 30
+
+# Maximum number of records to fetch per page
+# Socrata API supports up to 50000 records per request
+DEFAULT_PAGE_SIZE = 50000

omniload/src/solidgate/__init__.py

@@ -0,0 +1,219 @@
+from typing import Iterable, Iterator
+
+import dlt
+import pendulum
+from dlt.sources import DltResource
+
+from .helpers import SolidgateClient
+
+COLUMN_HINTS = {
+    "subscriptions": {
+        "id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "status": {"data_type": "text"},
+        "started_at": {"data_type": "timestamp"},
+        "updated_at": {"data_type": "timestamp"},
+        "expired_at": {"data_type": "timestamp"},
+        "next_charge_at": {"data_type": "timestamp"},
+        "payment_type": {"data_type": "text"},
+        "trial": {"data_type": "bool"},
+        "cancelled_at": {"data_type": "timestamp"},
+        "cancellation_requested_at": {"data_type": "timestamp"},
+        "cancel_code": {"data_type": "text"},
+        "cancel_message": {"data_type": "text"},
+        "customer": {"data_type": "json"},
+        "product": {"data_type": "json"},
+        "invoices": {"data_type": "json"},
+    },
+    "apm_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "method": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+    },
+    "card_orders": {
+        "order_id": {"data_type": "text", "nullable": False, "primary_key": True},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "updated_at": {"data_type": "timestamp"},
+        "order_description": {"data_type": "text"},
+        "psp_order_id": {"data_type": "text"},
+        "provider_payment_id": {"data_type": "text"},
+        "amount": {"data_type": "bigint"},
+        "currency": {"data_type": "text"},
+        "processing_amount": {"data_type": "bigint"},
+        "processing_currency": {"data_type": "text"},
+        "status": {"data_type": "text"},
+        "payment_type": {"data_type": "text"},
+        "type": {"data_type": "text"},
+        "is_secured": {"data_type": "bool"},
+        "routing": {"data_type": "json"},
+        "customer_account_id": {"data_type": "text"},
+        "customer_email": {"data_type": "text"},
+        "customer_first_name": {"data_type": "text"},
+        "customer_last_name": {"data_type": "text"},
+        "ip_address": {"data_type": "text"},
+        "mid": {"data_type": "text"},
+        "traffic_source": {"data_type": "text"},
+        "platform": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "error_code": {"data_type": "text"},
+        "transactions": {"data_type": "json"},
+        "order_metadata": {"data_type": "json"},
+        "fraudulent": {"data_type": "bool"},
+    },
+    "financial_entries": {
+        "id": {
+            "data_type": "text",
+            "nullable": False,
+            "primary_key": True,
+        },
+        "order_id": {"data_type": "text"},
+        "external_psp_order_id": {"data_type": "text"},
+        "created_at": {"data_type": "timestamp", "partition": True},
+        "transaction_datetime_provider": {"data_type": "timestamp"},
+        "transaction_datetime_utc": {"data_type": "timestamp"},
+        "accounting_date": {"data_type": "date"},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
+        "currency": {"data_type": "text"},
+        "currency_minor_units": {"data_type": "bigint"},
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
+        "payout_currency": {"data_type": "text"},
+        "payout_currency_minor_units": {"data_type": "bigint"},
+        "record_type_key": {"data_type": "text"},
+        "provider": {"data_type": "text"},
+        "payment_method": {"data_type": "text"},
+        "card_brand": {"data_type": "text"},
+        "geo_country": {"data_type": "text"},
+        "issuing_country": {"data_type": "text"},
+        "transaction_id": {"data_type": "text"},
+        "chargeback_id": {"data_type": "text"},
+        "legal_entity": {"data_type": "text"},
+    },
+}
+
+
+@dlt.source(max_table_nesting=0)
+def solidgate_source(
+    start_date: pendulum.DateTime,
+    end_date: pendulum.DateTime | None,
+    public_key: str,
+    secret_key: str,
+) -> Iterable[DltResource]:
+    solidgate_client = SolidgateClient(public_key, secret_key)
+
+    @dlt.resource(
+        name="subscriptions",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["subscriptions"],  # type: ignore
+    )
+    def fetch_all_subscriptions(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "subscriptions"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="apm_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["apm_orders"],  # type: ignore
+    )
+    def fetch_apm_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "apm-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="card_orders",
+        write_disposition="merge",
+        primary_key="order_id",
+        columns=COLUMN_HINTS["card_orders"],  # type: ignore
+    )
+    def fetch_card_orders(
+        dateTime=dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[dict]:
+        path = "card-orders"
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+        yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
+
+    @dlt.resource(
+        name="financial_entries",
+        write_disposition="merge",
+        primary_key="id",
+        columns=COLUMN_HINTS["financial_entries"],  # type: ignore
+    )
+    def fetch_financial_entries(
+        dateTime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ):
+        if dateTime.end_value is None:
+            end_date = pendulum.now(tz="UTC")
+        else:
+            end_date = dateTime.end_value
+
+        start_date = dateTime.last_value
+        yield solidgate_client.fetch_financial_entry_data(start_date, end_date)
+
+    return (
+        fetch_all_subscriptions,
+        fetch_apm_orders,
+        fetch_card_orders,
+        fetch_financial_entries,
+    )

omniload/src/solidgate/helpers.py

@@ -0,0 +1,154 @@
+import base64
+import csv
+import hashlib
+import hmac
+import json
+import time
+from io import StringIO
+
+import pendulum
+
+from omniload.src.http_client import create_client
+
+
+class SolidgateClient:
+    def __init__(self, public_key, secret_key):
+        self.base_url = "https://reports.solidgate.com/api/v1"
+        self.public_key = public_key
+        self.secret_key = secret_key
+        self.client = create_client(retry_status_codes=[204])
+
+    def fetch_data(
+        self,
+        path: str,
+        date_from: pendulum.DateTime,
+        date_to: pendulum.DateTime,
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        next_page_iterator = None
+        url = f"{self.base_url}/{path}"
+
+        while True:
+            payload = request_payload.copy()
+            if next_page_iterator:
+                payload["page_iterator"] = next_page_iterator
+
+            response = self.client.post(url, headers=headers, json=payload)
+            response.raise_for_status()
+            response_json = response.json()
+
+            if path == "subscriptions":
+                data = response_json["subscriptions"]
+                for _, value in data.items():
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
+                    yield value
+
+            else:
+                data = response_json["orders"]
+                for value in data:
+                    if "updated_at" in value:
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
+                    yield value
+
+            next_page_iterator = response_json.get("metadata", {}).get(
+                "next_page_iterator"
+            )
+            if not next_page_iterator or next_page_iterator == "None":
+                break
+
+    def fetch_financial_entry_data(
+        self, date_from: pendulum.DateTime, date_to: pendulum.DateTime
+    ):
+        request_payload = {
+            "date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
+            "date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
+        }
+
+        json_string = json.dumps(request_payload)
+        signature = self.generateSignature(json_string)
+        headers = {
+            "merchant": self.public_key,
+            "Signature": signature,
+            "Content-Type": "application/json",
+        }
+
+        url = f"{self.base_url}/finance/financial_entries"
+        post_response = self.client.post(url, headers=headers, json=request_payload)
+        post_response.raise_for_status()
+        report_url = post_response.json().get("report_url")
+        if not report_url:
+            return f"Report URL not found in the response: {post_response.json()}", 400
+
+        data = self.public_key + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        signature_get = base64.b64encode(hmac_hash.hex().encode("utf-8")).decode(
+            "utf-8"
+        )
+
+        headers_get = {
+            "merchant": self.public_key,
+            "Signature": signature_get,
+            "Content-Type": "application/json",
+        }
+
+        # Retry getting the report for up to 10 minutes (600 seconds) with 5-second intervals
+        max_retries = 120  # 10 minutes / 5 seconds = 120 attempts
+        retry_count = 0
+
+        while retry_count < max_retries:
+            get_response = self.client.get(report_url, headers=headers_get)
+
+            if get_response.status_code == 200:
+                try:
+                    response_json = json.loads(get_response.content)
+                    if "error" in response_json:
+                        raise Exception(
+                            f"API Error: {response_json['error']['messages']}"
+                        )
+                except json.JSONDecodeError:
+                    try:
+                        csv_data = get_response.content.decode("utf-8")
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ""}
+                            rows.append(row2)
+
+                        return rows
+                    except Exception as e:
+                        raise Exception(f"Error reading CSV: {e}")
+            else:
+                # Report might not be ready yet, wait and retry
+                retry_count += 1
+                if retry_count >= max_retries:
+                    raise Exception(
+                        f"Failed to get report after {max_retries} attempts. Status code: {get_response.status_code}"
+                    )
+                time.sleep(5)  # Wait 5 seconds before retrying
+
+    def generateSignature(self, json_string):
+        data = self.public_key + json_string + self.public_key
+        hmac_hash = hmac.new(
+            self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
+        ).digest()
+        return base64.b64encode(hmac_hash.hex().encode("utf-8")).decode("utf-8")