omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/reddit_ads/helpers.py

@@ -0,0 +1,232 @@
+import time
+
+import requests
+from dlt.sources.helpers.requests import Client
+from pendulum import Date
+
+MONETARY_FIELDS = {"spend", "ecpm", "cpc"}
+
+BASE_URL = "https://ads-api.reddit.com/api/v3"
+
+LEVEL_ID_FIELDS = {
+    "ACCOUNT": "account_id",
+    "CAMPAIGN": "campaign_id",
+    "AD_GROUP": "ad_group_id",
+    "AD": "ad_id",
+}
+
+VALID_LEVELS = {"ACCOUNT", "CAMPAIGN", "AD_GROUP", "AD"}
+
+VALID_BREAKDOWNS = {
+    "date",
+    "country",
+    "region",
+    "community",
+    "placement",
+    "device_os",
+    "gender",
+    "interest",
+    "keyword",
+    "carousel_card",
+}
+
+VALID_METRICS = {
+    "IMPRESSIONS",
+    "REACH",
+    "CLICKS",
+    "SPEND",
+    "ECPM",
+    "CTR",
+    "CPC",
+    "CONVERSIONS",
+    "CONVERSION_ROAS",
+    "TOTAL_ITEMS",
+    "TOTAL_VALUE",
+    "AVG_VALUE",
+    "REDDIT_LEADS",
+    "COMMENTS_PAGE_VIEWS",
+    "COMMENT_UPVOTES",
+    "COMMENT_DOWNVOTES",
+    "VIEWER_COMMENTS",
+    "VIDEO_STARTED",
+    "VIDEO_WATCHED_3_SECONDS",
+    "VIDEO_WATCHED_5_SECONDS",
+    "VIDEO_WATCHED_25_PERCENT",
+    "VIDEO_WATCHED_50_PERCENT",
+    "VIDEO_WATCHED_75_PERCENT",
+    "VIDEO_WATCHED_100_PERCENT",
+    "VIDEO_WATCHED_6_SECONDS_RATE",
+    "VIDEO_WATCHED_15_SECONDS_RATE",
+}
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+def handle_rate_limit(response: requests.Response) -> None:
+    remaining = response.headers.get("X-RateLimit-Remaining")
+    reset = response.headers.get("X-RateLimit-Reset")
+    if remaining is not None and reset is not None:
+        try:
+            if float(remaining) < 2:
+                sleep_time = float(reset)
+                if sleep_time > 0:
+                    time.sleep(sleep_time)
+        except (ValueError, TypeError):
+            pass
+
+
+def convert_microcurrency(records: list[dict], metrics: list[str]) -> list[dict]:
+    monetary = MONETARY_FIELDS & {m.lower() for m in metrics}
+    if not monetary:
+        return records
+    for record in records:
+        for field in monetary:
+            if field in record and record[field] is not None:
+                record[field] = record[field] / 1_000_000
+    return records
+
+
+def parse_custom_table(table: str) -> tuple[str, list[str], list[str]]:
+    parts = table.split(":")
+    if len(parts) != 3:
+        raise ValueError(
+            "Invalid custom table format. Expected: custom:<level>,<breakdowns>:<metrics>"
+        )
+
+    dimensions = [d.strip() for d in parts[1].split(",") if d.strip()]
+    if not dimensions:
+        raise ValueError("At least a level is required in the dimensions segment")
+
+    level = dimensions[0].upper()
+    if level not in VALID_LEVELS:
+        raise ValueError(
+            f"Invalid level '{level}'. Must be one of: {', '.join(sorted(VALID_LEVELS))}"
+        )
+
+    breakdowns = [b.lower() for b in dimensions[1:]]
+    if len(breakdowns) > 2:
+        raise ValueError("Reddit Ads supports at most 2 breakdowns per report")
+
+    for b in breakdowns:
+        if b not in VALID_BREAKDOWNS:
+            raise ValueError(
+                f"Invalid breakdown '{b}'. Must be one of: {', '.join(sorted(VALID_BREAKDOWNS))}"
+            )
+
+    metrics = [m.strip().upper() for m in parts[2].split(",") if m.strip()]
+    if not metrics:
+        raise ValueError("At least one metric is required")
+
+    for m in metrics:
+        if m not in VALID_METRICS:
+            raise ValueError(
+                f"Invalid metric '{m}'. Must be one of: {', '.join(sorted(VALID_METRICS))}"
+            )
+
+    return level, breakdowns, metrics
+
+
+class RedditAdsAPI:
+    def __init__(self, access_token: str):
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "User-Agent": "omniload/1.0",
+        }
+        self.client = create_client()
+
+    def fetch_pages(self, url: str, page_size: int = 100):
+        separator = "&" if "?" in url else "?"
+        paginated_url = f"{url}{separator}page_size={page_size}"
+
+        while True:
+            response = self.client.get(url=paginated_url, headers=self.headers)
+
+            if response.status_code != 200:
+                raise ValueError(
+                    f"Reddit Ads API Error ({response.status_code}): {response.text}"
+                )
+
+            handle_rate_limit(response)
+
+            result = response.json()
+            elements = result.get("data", [])
+
+            if not elements:
+                break
+
+            yield elements
+
+            pagination = result.get("pagination", {})
+            next_url = pagination.get("next_url")
+            if not next_url:
+                break
+
+            paginated_url = next_url
+
+
+class RedditAdsReportAPI:
+    def __init__(
+        self,
+        access_token: str,
+        account_ids: list[str],
+        level: str,
+        breakdowns: list[str],
+        metrics: list[str],
+    ):
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "User-Agent": "omniload/1.0",
+            "Content-Type": "application/json",
+        }
+        self.account_ids = account_ids
+        self.level = level
+        self.breakdowns = breakdowns
+        self.metrics = metrics
+        self.client = create_client()
+
+    def fetch_report(self, start_date: Date, end_date: Date):
+        body = {
+            "start_date": start_date.to_date_string(),
+            "end_date": end_date.to_date_string(),
+            "level": self.level,
+            "metrics": self.metrics,
+            "breakdowns": self.breakdowns,
+        }
+
+        for account_id in self.account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/reports"
+            response = self.client.post(url=url, json=body, headers=self.headers)
+
+            if response.status_code != 200:
+                raise ValueError(
+                    f"Reddit Ads Report API Error ({response.status_code}): {response.text}"
+                )
+
+            handle_rate_limit(response)
+
+            result = response.json()
+            records = result.get("data", [])
+
+            if not records:
+                continue
+
+            for record in records:
+                record["account_id"] = account_id
+
+            records = convert_microcurrency(records, self.metrics)
+            yield records

omniload/src/resource.py

@@ -0,0 +1,40 @@
+from typing import Callable
+
+from dlt.sources import DltResource, DltSource
+
+
+def for_each(
+    source: DltSource | DltResource, ex: Callable[[DltResource], None | DltResource]
+):
+    """
+    Apply a function to each resource in a source.
+    """
+    if hasattr(source, "selected_resources") and source.selected_resources:
+        resource_names = list(source.selected_resources.keys())
+        for res in resource_names:
+            ex(source.resources[res])  # type: ignore[union-attr]
+    else:
+        ex(source)  # type: ignore[arg-type]
+
+
+class TypeHintMap:
+    def __init__(self):
+        self.handled_typehints = False
+
+    def type_hint_map(self, item):
+        if self.handled_typehints:
+            return item
+
+        array_cols = []
+        for col in item:
+            if isinstance(item[col], (list, tuple)):
+                array_cols.append(col)
+        if array_cols:
+            import dlt
+
+            source = dlt.current.source()
+            columns = [{"name": col, "data_type": "json"} for col in array_cols]
+            for_each(source, lambda x: x.apply_hints(columns=columns))
+
+        self.handled_typehints = True
+        return item

omniload/src/revenuecat/__init__.py

@@ -0,0 +1,83 @@
+from typing import Any, Dict, Iterable, Iterator
+
+import aiohttp
+import dlt
+
+from .helpers import (
+    _make_request,
+    _paginate,
+    convert_timestamps_to_iso,
+    create_project_resource,
+    process_customer_with_nested_resources_async,
+)
+
+
+@dlt.source(name="revenuecat", max_table_nesting=0)
+def revenuecat_source(
+    api_key: str,
+    project_id: str = None,
+) -> Iterable[dlt.sources.DltResource]:
+    """
+    RevenueCat source for extracting data from RevenueCat API v2.
+
+    Args:
+        api_key: RevenueCat API v2 secret key with Bearer token format
+        project_id: RevenueCat project ID (required for customers, products, entitlements, offerings, subscriptions, purchases)
+
+    Returns:
+        Iterable of DLT resources for customers, products, entitlements, offerings, purchases, subscriptions, and projects
+    """
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects() -> Iterator[Dict[str, Any]]:
+        """Get list of projects."""
+        # Get projects list
+        data = _make_request(api_key, "/projects")
+        if "items" in data:
+            for project in data["items"]:
+                project = convert_timestamps_to_iso(project, ["created_at"])
+                yield project
+
+    @dlt.resource(
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
+    )
+    def customer_ids():
+        if project_id is None:
+            raise ValueError("project_id is required for customers resource")
+
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
+
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
+
+    # Create project-dependent resources dynamically
+    project_resources = []
+    resource_names = ["products", "entitlements", "offerings"]
+
+    for resource_name in resource_names:
+
+        @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+        def create_resource(resource_name=resource_name) -> Iterator[Dict[str, Any]]:
+            """Get list of project resource."""
+            yield from create_project_resource(resource_name, api_key, project_id)
+
+        # Set the function name for better identification
+        create_resource.__name__ = resource_name
+        project_resources.append(create_resource)
+
+    return [
+        projects,
+        customer_ids,
+        customers,
+        *project_resources,
+    ]

omniload/src/revenuecat/helpers.py

@@ -0,0 +1,237 @@
+import asyncio
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+import aiohttp
+import pendulum
+import requests
+
+REVENUECAT_API_BASE = "https://api.revenuecat.com/v2"
+
+
+def _make_request(
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make a REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params or {})
+
+            # Handle rate limiting (429 Too Many Requests)
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    # Wait based on Retry-After header or exponential backoff
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                    time.sleep(wait_time)
+                    continue
+
+            response.raise_for_status()
+            return response.json()
+
+        except requests.exceptions.RequestException:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                time.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    response.raise_for_status()
+    return response.json()
+
+
+def _paginate(
+    api_key: str, endpoint: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Paginate through RevenueCat API results."""
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = _make_request(api_key, endpoint, current_params)
+
+        if "items" in data and data["items"] is not None:
+            yield data["items"]
+
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+
+def convert_timestamps_to_iso(
+    record: Dict[str, Any], timestamp_fields: List[str]
+) -> Dict[str, Any]:
+    """Convert timestamp fields from milliseconds to ISO format."""
+    for field in timestamp_fields:
+        if field in record and record[field] is not None:
+            timestamp_ms = record[field]
+            dt = pendulum.from_timestamp(timestamp_ms / 1000)
+            record[field] = dt.to_iso8601_string()
+
+    return record
+
+
+async def _make_request_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make an async REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            async with session.get(
+                url, headers=headers, params=params or {}
+            ) as response:
+                # Handle rate limiting (429 Too Many Requests)
+                if response.status == 429:
+                    if attempt < max_retries:
+                        # Wait based on Retry-After header or exponential backoff
+                        retry_after = response.headers.get("Retry-After")
+                        if retry_after:
+                            wait_time = int(retry_after)
+                        else:
+                            wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                response.raise_for_status()
+                return await response.json()
+
+        except aiohttp.ClientError:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                await asyncio.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    async with session.get(url, headers=headers, params=params or {}) as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """Paginate through RevenueCat API results asynchronously."""
+    items = []
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = await _make_request_async(session, api_key, endpoint, current_params)
+
+        # Collect items from the current page
+        if "items" in data and data["items"] is not None:
+            items.extend(data["items"])
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+    return items
+
+
+async def process_customer_with_nested_resources_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer: Dict[str, Any],
+) -> Dict[str, Any]:
+    customer_id = customer["id"]
+    customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
+    nested_resources = [
+        ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
+        ("purchases", ["purchased_at", "expires_at"]),
+    ]
+
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
+
+    return customer
+
+
+def create_project_resource(
+    resource_name: str,
+    api_key: str,
+    project_id: str = None,
+    timestamp_fields: List[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to create DLT resources for project-dependent endpoints.
+
+    Args:
+        resource_name: Name of the resource (e.g., 'products', 'entitlements', 'offerings')
+        api_key: RevenueCat API key
+        project_id: RevenueCat project ID
+        timestamp_fields: List of timestamp fields to convert to ISO format
+
+    Returns:
+        Iterator of resource data
+    """
+    if project_id is None:
+        raise ValueError(f"project_id is required for {resource_name} resource")
+
+    endpoint = f"/projects/{project_id}/{resource_name}"
+    default_timestamp_fields = timestamp_fields or ["created_at", "updated_at"]
+
+    for item in _paginate(api_key, endpoint):
+        item = convert_timestamps_to_iso(item, default_timestamp_fields)
+        yield item