omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/linear/helpers.py

@@ -0,0 +1,111 @@
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+import pendulum
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+def _get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = pendulum.parse(start_date)
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
+
+def _paginated_resource(
+    api_key: str, query: str, query_field: str, updated_at, start_date
+) -> Iterator[Dict[str, Any]]:
+    """Helper function for paginated resources with date filtering."""
+    current_start_date, current_end_date = _get_date_range(updated_at, start_date)
+
+    for item in _paginate(api_key, query, query_field):
+        if pendulum.parse(item["updatedAt"]) >= current_start_date:
+            if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                yield normalize_dictionaries(item)
+
+
+def _create_paginated_resource(
+    resource_name: str,
+    query: str,
+    query_field: str,
+    api_key: str,
+    start_date,
+    end_date=None,
+):
+    """Factory function to create paginated resources dynamically."""
+
+    @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+    def paginated_resource(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        for item in _paginated_resource(
+            api_key, query, query_field, updated_at, start_date
+        ):
+            yield normalize_dictionaries(item)
+
+    return paginated_resource
+
+
+def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Automatically normalize dictionary fields by detecting their structure:
+    - Convert nested objects with 'id' field to {field_name}_id
+    - Convert objects with 'nodes' field to arrays
+
+    """
+    normalized_item = item.copy()
+
+    for key, value in list(normalized_item.items()):
+        if isinstance(value, dict):
+            # If the dict has an 'id' field, replace with {key}_id
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
+                del normalized_item[key]
+            # If the dict has 'nodes' field, extract the nodes array
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
+    return normalized_item

omniload/src/linkedin_ads/__init__.py

@@ -0,0 +1,266 @@
+from typing import Iterable
+from urllib.parse import quote
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from pendulum import Date, DateTime
+
+from .dimension_time_enum import Dimension, TimeGranularity
+from .helpers import LinkedInAdsAnalyticsAPI, LinkedInAdsAPI, find_intervals
+
+
+@dlt.source(max_table_nesting=0)
+def linked_in_ads_analytics_source(
+    start_date: Date,
+    end_date: Date | None,
+    access_token: str,
+    account_ids: list[str],
+    dimension: Dimension,
+    metrics: list[str],
+    time_granularity: TimeGranularity,
+) -> DltResource:
+    if time_granularity == TimeGranularity.daily:
+        primary_key = [dimension.value, "date"]
+        incremental_loading_param = "date"
+    else:
+        primary_key = [dimension.value, "start_date", "end_date"]
+        incremental_loading_param = "start_date"
+
+    @dlt.resource(write_disposition="merge", primary_key=primary_key)
+    def custom_reports(
+        dateTime=(
+            dlt.sources.incremental(
+                incremental_loading_param,
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        linkedin_api = LinkedInAdsAnalyticsAPI(
+            access_token=access_token,
+            account_ids=account_ids,
+            dimension=dimension,
+            metrics=metrics,
+            time_granularity=time_granularity,
+        )
+
+        if dateTime.end_value is None:
+            end_date = pendulum.now().date()
+        else:
+            end_date = dateTime.end_value
+
+        list_of_interval = find_intervals(
+            start_date=dateTime.last_value,
+            end_date=end_date,
+            time_granularity=time_granularity,
+        )
+        for start, end in list_of_interval:
+            yield linkedin_api.fetch_pages(start, end)
+
+    return custom_reports
+
+
+@dlt.source(max_table_nesting=0)
+def linked_in_ads_source(
+    access_token: str, start_datetime: DateTime, end_datetime: DateTime | None
+) -> list[DltResource]:
+    linkedin_api = LinkedInAdsAPI(
+        access_token=access_token,
+    )
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def ad_accounts() -> Iterable[TDataItem]:
+        yield from linkedin_api.fetch_token_pagination(
+            url="https://api.linkedin.com/rest/adAccounts?q=search"
+        )
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key=["user", "account"],
+        data_from=ad_accounts,
+    )
+    def ad_account_users(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/adAccountUsers?q=accounts&accounts=List({encoded_id})"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def campaign_groups(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            url = f"https://api.linkedin.com/rest/adAccounts/{account_id}/adCampaignGroups?q=search"
+            for page in linkedin_api.fetch_token_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def campaigns(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            url = f"https://api.linkedin.com/rest/adAccounts/{account_id}/adCampaigns?q=search"
+            for page in linkedin_api.fetch_token_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def creatives(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            url = f"https://api.linkedin.com/rest/adAccounts/{account_id}/creatives?q=criteria"
+            for page in linkedin_api.fetch_token_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def conversions(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/conversions?q=account&account={encoded_id}"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def lead_forms(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/leadForms?q=owner&owner=(sponsoredAccount:{encoded_id})"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="merge",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def lead_form_responses(
+        ad_accounts,
+        submittedAt=dlt.sources.incremental(
+            "submittedAt",
+            initial_value=int(start_datetime.int_timestamp * 1000),
+            end_value=end_datetime.int_timestamp * 1000 if end_datetime else None,
+            range_end="closed" if end_datetime else "open",
+            range_start="closed",
+        ),
+    ) -> Iterable[TDataItem]:
+        fromDate = submittedAt.start_value
+        toDate = (
+            submittedAt.end_value
+            if submittedAt.end_value
+            else int(pendulum.now(tz="UTC").int_timestamp * 1000)
+        )
+
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/leadFormResponses?leadType=(leadType:SPONSORED)&q=owner&owner=(sponsoredAccount:{encoded_id})&submittedAtTimeRange=(start:{fromDate},end:{toDate})&limitedToTestLeads=false"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def dmp_segments(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/dmpSegments?q=account&account={encoded_id}"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key="id",
+        data_from=ad_accounts,
+    )
+    def insight_tags(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/insightTags?q=account&account={encoded_id}"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    @dlt.transformer(
+        write_disposition="replace",
+        primary_key=["domainName", "account_id"],
+        data_from=ad_accounts,
+    )
+    def insight_tag_domains(ad_accounts) -> Iterable[TDataItem]:
+        for ad_account in ad_accounts:
+            account_id = ad_account["id"]
+            encoded_id = quote(f"urn:li:sponsoredAccount:{account_id}")
+            url = f"https://api.linkedin.com/rest/insightTagDomains?q=account&account={encoded_id}"
+            for page in linkedin_api.fetch_cursor_pagination(url):
+                for item in page:
+                    item["account_id"] = account_id
+
+                yield page
+
+    return [
+        ad_accounts,
+        ad_account_users,
+        campaign_groups,
+        campaigns,
+        creatives,
+        conversions,
+        lead_forms,
+        lead_form_responses,
+        dmp_segments,
+        insight_tags,
+        insight_tag_domains,
+    ]

omniload/src/linkedin_ads/dimension_time_enum.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+
+class Dimension(Enum):
+    campaign = "campaign"
+    creative = "creative"
+    account = "account"
+    member_job_title = "member_job_title"
+    member_seniority = "member_seniority"
+    member_industry = "member_industry"
+    member_company_size = "member_company_size"
+    member_company = "member_company"
+
+
+class TimeGranularity(Enum):
+    daily = "DAILY"
+    monthly = "MONTHLY"

omniload/src/linkedin_ads/helpers.py

@@ -0,0 +1,246 @@
+from urllib.parse import quote
+
+import pendulum
+import requests
+from dlt.sources.helpers.requests import Client
+from pendulum import Date
+
+from .dimension_time_enum import Dimension, TimeGranularity
+
+
+def retry_on_limit(
+    response: requests.Response | None, exception: BaseException | None
+) -> bool:
+    if response is None:
+        return False
+    return response.status_code == 429
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+    ).session
+
+
+def flat_structure(items, pivot: Dimension, time_granularity: TimeGranularity):
+    for item in items:
+        if "pivotValues" in item:
+            if len(item["pivotValues"]) > 1:
+                item[pivot.value.lower()] = item["pivotValues"]
+            else:
+                item[pivot.value.lower()] = item["pivotValues"][0]
+        if "dateRange" in item:
+            start_date = item["dateRange"]["start"]
+            start_dt = pendulum.date(
+                year=start_date["year"],
+                month=start_date["month"],
+                day=start_date["day"],
+            )
+            if time_granularity == TimeGranularity.daily:
+                item["date"] = start_dt
+            else:
+                end_date = item["dateRange"]["end"]
+                end_dt = pendulum.date(
+                    year=end_date["year"],
+                    month=end_date["month"],
+                    day=end_date["day"],
+                )
+                item["start_date"] = start_dt
+                item["end_date"] = end_dt
+
+        del item["dateRange"]
+        del item["pivotValues"]
+
+    return items
+
+
+def find_intervals(start_date: Date, end_date: Date, time_granularity: TimeGranularity):
+    intervals = []
+
+    if start_date > end_date:
+        raise ValueError("Start date must be less than end date")
+
+    while start_date <= end_date:
+        if time_granularity == TimeGranularity.daily:
+            next_date = min(start_date.add(months=6), end_date)
+        else:
+            next_date = min(start_date.add(years=2), end_date)
+
+        intervals.append((start_date, next_date))
+
+        start_date = next_date.add(days=1)
+
+    return intervals
+
+
+DIMENSION_PIVOT_MAP = {
+    "campaign": "CAMPAIGN",
+    "creative": "CREATIVE",
+    "account": "ACCOUNT",
+    "member_job_title": "MEMBER_JOB_TITLE",
+    "member_seniority": "MEMBER_SENIORITY",
+    "member_industry": "MEMBER_INDUSTRY",
+    "member_company_size": "MEMBER_COMPANY_SIZE",
+    "member_company": "MEMBER_COMPANY",
+}
+
+
+def construct_url(
+    start: Date,
+    end: Date,
+    account_ids: list[str],
+    metrics: list[str],
+    dimension: Dimension,
+    time_granularity: TimeGranularity,
+):
+    date_range = f"(start:(year:{start.year},month:{start.month},day:{start.day})"
+    date_range += f",end:(year:{end.year},month:{end.month},day:{end.day}))"
+    accounts = ",".join(
+        [quote(f"urn:li:sponsoredAccount:{account_id}") for account_id in account_ids]
+    )
+    encoded_accounts = f"List({accounts})"
+    dimension_str = DIMENSION_PIVOT_MAP[dimension.value]
+    time_granularity_str = time_granularity.value
+    metrics_str = ",".join([metric for metric in metrics])
+
+    url = (
+        f"https://api.linkedin.com/rest/adAnalytics?"
+        f"q=analytics&timeGranularity={time_granularity_str}&"
+        f"dateRange={date_range}&accounts={encoded_accounts}&"
+        f"pivot={dimension_str}&fields={metrics_str}"
+    )
+
+    return url
+
+
+class LinkedInAdsAnalyticsAPI:
+    def __init__(
+        self,
+        access_token,
+        time_granularity,
+        account_ids,
+        dimension,
+        metrics,
+    ):
+        self.time_granularity: TimeGranularity = time_granularity
+        self.account_ids: list[str] = account_ids
+        self.dimension: Dimension = dimension
+        self.metrics: list[str] = metrics
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Linkedin-Version": "202601",
+            "X-Restli-Protocol-Version": "2.0.0",
+        }
+
+    def fetch_pages(self, start: Date, end: Date):
+        client = create_client()
+        url = construct_url(
+            start=start,
+            end=end,
+            account_ids=self.account_ids,
+            metrics=self.metrics,
+            dimension=self.dimension,
+            time_granularity=self.time_granularity,
+        )
+        response = client.get(url=url, headers=self.headers)
+
+        if response.status_code != 200:
+            error_data = response.json()
+            raise ValueError(f"LinkedIn API Error: {error_data.get('message')}")
+
+        result = response.json()
+        items = result.get("elements", [])
+        yield flat_structure(
+            items=items,
+            pivot=self.dimension,
+            time_granularity=self.time_granularity,
+        )
+
+
+class LinkedInAdsAPI:
+    def __init__(
+        self,
+        access_token,
+    ):
+        self.headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Linkedin-Version": "202601",
+            "X-Restli-Protocol-Version": "2.0.0",
+        }
+        self.client = create_client()
+
+    def fetch_full(self, url: str):
+        response = self.client.get(url=url, headers=self.headers)
+
+        if response.status_code != 200:
+            error_data = response.json()
+            raise ValueError(f"LinkedIn API Error: {error_data}")
+
+        result = response.json()
+        elements = result.get("elements", [])
+
+        if elements:
+            yield elements
+
+    def fetch_token_pagination(self, url: str, page_size: int = 1000):
+        next_page_token = None
+        separator = "&" if "?" in url else "?"
+
+        while True:
+            if next_page_token:
+                paginated_url = (
+                    f"{url}{separator}pageSize={page_size}&pageToken={next_page_token}"
+                )
+            else:
+                paginated_url = f"{url}{separator}pageSize={page_size}"
+
+            response = self.client.get(url=paginated_url, headers=self.headers)
+
+            if response.status_code != 200:
+                error_data = response.json()
+                raise ValueError(f"LinkedIn API Error: {error_data}")
+
+            result = response.json()
+            elements = result.get("elements", [])
+
+            if not elements:
+                break
+
+            yield elements
+
+            if len(elements) < page_size:
+                break
+
+            metadata = result.get("metadata", {})
+            next_page_token = metadata.get("nextPageToken")
+
+            if not next_page_token:
+                break
+
+    def fetch_cursor_pagination(self, url: str, count: int = 1000):
+        start = 0
+        separator = "&" if "?" in url else "?"
+
+        while True:
+            paginated_url = f"{url}{separator}start={start}&count={count}"
+
+            response = self.client.get(url=paginated_url, headers=self.headers)
+
+            if response.status_code != 200:
+                error_data = response.json()
+                raise ValueError(f"LinkedIn API Error: {error_data}")
+
+            result = response.json()
+            elements = result.get("elements", [])
+
+            if not elements:
+                break
+
+            yield elements
+
+            if len(elements) < count:
+                break
+
+            start += count