omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/plusvibeai/settings.py

@@ -0,0 +1,252 @@
+"""PlusVibeAI source settings and constants"""
+
+# Default start date for PlusVibeAI API requests
+DEFAULT_START_DATE = "2020-01-01"
+
+# PlusVibeAI API request timeout in seconds
+REQUEST_TIMEOUT = 300
+
+# Default page size for paginated requests
+DEFAULT_PAGE_SIZE = 100
+
+# Maximum page size (adjust based on API limits)
+MAX_PAGE_SIZE = 1000
+
+# Base API path for PlusVibeAI
+API_BASE_PATH = "/api/v1"
+
+# Campaign fields to retrieve from PlusVibeAI API
+CAMPAIGN_FIELDS = (
+    # Basic Information
+    "id",
+    "camp_name",
+    "parent_camp_id",
+    "campaign_type",
+    "organization_id",
+    "workspace_id",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_lead_sent",
+    "last_paused_at_bounced",
+    # Campaign Configuration
+    "tags",
+    "template_id",
+    "email_accounts",
+    "daily_limit",
+    "interval_limit_in_min",
+    "send_priority",
+    "send_as_txt",
+    # Tracking & Settings
+    "is_emailopened_tracking",
+    "is_unsubscribed_link",
+    "exclude_ooo",
+    "is_acc_based_sending",
+    "send_risky_email",
+    "unsub_blocklist",
+    "other_email_acc",
+    "is_esp_match",
+    "stop_on_lead_replied",
+    # Bounce Settings
+    "is_pause_on_bouncerate",
+    "bounce_rate_limit",
+    "is_paused_at_bounced",
+    # Schedule
+    "schedule",
+    "first_wait_time",
+    "camp_st_date",
+    "camp_end_date",
+    # Events & Sequences
+    "events",
+    "sequences",
+    "sequence_steps",
+    "camp_emails",
+    # Lead Statistics
+    "lead_count",
+    "completed_lead_count",
+    "lead_contacted_count",
+    # Email Performance Metrics
+    "sent_count",
+    "opened_count",
+    "unique_opened_count",
+    "replied_count",
+    "bounced_count",
+    "unsubscribed_count",
+    # Reply Classification
+    "positive_reply_count",
+    "negative_reply_count",
+    "neutral_reply_count",
+    # Daily & Business Metrics
+    "email_sent_today",
+    "opportunity_val",
+    "open_rate",
+    "replied_rate",
+    # Custom Data
+    "custom_fields",
+)
+
+# Lead fields to retrieve from PlusVibeAI API
+LEAD_FIELDS = (
+    # Basic Information
+    "_id",
+    "organization_id",
+    "campaign_id",
+    "workspace_id",
+    # Lead Status & Progress
+    "is_completed",
+    "current_step",
+    "status",
+    "label",
+    # Email Account Info
+    "email_account_id",
+    "email_acc_name",
+    # Campaign Info
+    "camp_name",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_sent_at",
+    # Email Engagement Metrics
+    "sent_step",
+    "replied_count",
+    "opened_count",
+    # Email Verification
+    "is_mx",
+    "mx",
+    # Contact Information
+    "email",
+    "first_name",
+    "last_name",
+    "phone_number",
+    # Address Information
+    "address_line",
+    "city",
+    "state",
+    "country",
+    "country_code",
+    # Professional Information
+    "job_title",
+    "department",
+    "company_name",
+    "company_website",
+    "industry",
+    # Social Media
+    "linkedin_person_url",
+    "linkedin_company_url",
+    # Workflow
+    "total_steps",
+    # Bounce Information
+    "bounce_msg",
+)
+
+# Email Account fields to retrieve from PlusVibeAI API
+EMAIL_ACCOUNT_FIELDS = (
+    # Basic Information
+    "_id",
+    "email",
+    "status",
+    "warmup_status",
+    # Timestamps
+    "timestamp_created",
+    "timestamp_updated",
+    # Payload - nested object containing all configuration
+    "payload",
+    # Payload sub-fields (for reference, stored in payload object):
+    # - name (first_name, last_name)
+    # - warmup (limit, warmup_custom_words, warmup_signature, advanced, increment, reply_rate)
+    # - imap_host, imap_port
+    # - smtp_host, smtp_port
+    # - daily_limit, sending_gap
+    # - reply_to, custom_domain, signature
+    # - tags, cmps
+    # - analytics (health_scores, reply_rates, daily_counters)
+)
+
+# Email fields to retrieve from PlusVibeAI API
+EMAIL_FIELDS = (
+    # Basic Information
+    "id",
+    "message_id",
+    "is_unread",
+    # Lead Information
+    "lead",
+    "lead_id",
+    "campaign_id",
+    # From Address
+    "from_address_email",
+    "from_address_json",
+    # Subject & Content
+    "subject",
+    "content_preview",
+    "body",
+    # Headers & Metadata
+    "headers",
+    "label",
+    "thread_id",
+    "eaccount",
+    # To/CC/BCC Addresses
+    "to_address_email_list",
+    "to_address_json",
+    "cc_address_email_list",
+    "cc_address_json",
+    "bcc_address_email_list",
+    # Timestamps
+    "timestamp_created",
+    "source_modified_at",
+)
+
+# Blocklist fields to retrieve from PlusVibeAI API
+BLOCKLIST_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "value",
+    "created_by_label",
+    # Timestamps
+    "created_at",
+)
+
+# Webhook fields to retrieve from PlusVibeAI API
+WEBHOOK_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "url",
+    "name",
+    "secret",
+    # Configuration
+    "camp_ids",
+    "evt_types",
+    "status",
+    "integration_type",
+    # Settings
+    "ignore_ooo",
+    "ignore_automatic",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_run",
+    # Response Data
+    "last_resp",
+    "last_recv_resp",
+    # User Information
+    "created_by",
+    "modified_by",
+)
+
+# Tag fields to retrieve from PlusVibeAI API
+TAG_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "name",
+    "color",
+    "description",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+)

omniload/src/primer/__init__.py

@@ -0,0 +1,45 @@
+"""Fetches Primer payments data."""
+
+from typing import Iterable, Optional
+
+import dlt
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+
+from .helpers import PrimerApi
+
+
+@dlt.source(name="primer", max_table_nesting=0)
+def primer_source(
+    api_key: str = dlt.secrets.value,
+    api_version: str = "2.4",
+    start_date: Optional[TAnyDateTime] = None,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Iterable[DltResource]:
+    client = PrimerApi(api_key, api_version)
+
+    start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    @dlt.resource(selected=False)
+    def payment_ids() -> Iterable[TDataItem]:
+        """List payment IDs from Primer API."""
+        ids = client.list_payment_ids(
+            start_date=start_date_obj,
+            end_date=end_date_obj,
+        )
+        for payment_id in ids:
+            yield {"id": payment_id}
+
+    @dlt.transformer(
+        data_from=payment_ids,
+        primary_key="id",
+        write_disposition="merge",
+        parallelized=True,
+    )
+    def payments(payment_id_item: TDataItem) -> TDataItem:
+        """Fetch payment details in parallel."""
+        return client.get_payment(payment_id_item["id"])
+
+    return (payment_ids, payments)

omniload/src/primer/helpers.py

@@ -0,0 +1,79 @@
+"""Primer API client helpers"""
+
+from typing import Any, Dict, List, Optional
+
+from dlt.common.pendulum import pendulum
+from dlt.sources.helpers import requests
+
+
+def build_date_params(
+    start_date: Optional[pendulum.DateTime] = None,
+    end_date: Optional[pendulum.DateTime] = None,
+) -> Dict[str, str]:
+    params: Dict[str, str] = {}
+    if start_date:
+        params["from_date"] = start_date.start_of("day").to_iso8601_string()
+    if end_date:
+        params["to_date"] = end_date.add(days=1).start_of("day").to_iso8601_string()
+    return params
+
+
+class PrimerApi:
+    """
+    A Primer API client that handles pagination and data fetching.
+    """
+
+    BASE_URL = "https://api.primer.io"
+
+    def __init__(
+        self,
+        api_key: str,
+        api_version: str = "2.4",
+    ) -> None:
+        self.api_key = api_key
+        self.api_version = api_version
+
+    def _get_headers(self) -> Dict[str, str]:
+        return {
+            "X-API-KEY": self.api_key,
+            "X-API-VERSION": self.api_version,
+        }
+
+    def list_payment_ids(
+        self,
+        start_date: Optional[pendulum.DateTime] = None,
+        end_date: Optional[pendulum.DateTime] = None,
+    ) -> List[str]:
+        """List all payment IDs from the payments endpoint."""
+        url = f"{self.BASE_URL}/payments"
+        params: Dict[str, Any] = build_date_params(start_date, end_date)
+
+        payment_ids: List[str] = []
+
+        while True:
+            response = requests.get(url, params=params, headers=self._get_headers())
+            response.raise_for_status()
+
+            json_data = response.json()
+            data = json_data.get("data", [])
+
+            if len(data) == 0:
+                break
+
+            for item in data:
+                payment_ids.append(item["id"])
+
+            next_cursor = json_data.get("nextCursor")
+            if not next_cursor:
+                break
+
+            params["cursor"] = next_cursor
+
+        return payment_ids
+
+    def get_payment(self, payment_id: str) -> Dict[str, Any]:
+        """Get payment by ID."""
+        url = f"{self.BASE_URL}/payments/{payment_id}"
+        response = requests.get(url, headers=self._get_headers())
+        response.raise_for_status()
+        return response.json()

omniload/src/quickbooks/__init__.py

@@ -0,0 +1,117 @@
+"""QuickBooks source built on top of python-quickbooks."""
+
+from typing import Iterable, Iterator, List, Optional
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from intuitlib.client import AuthClient  # type: ignore
+
+from quickbooks import QuickBooks  # type: ignore
+
+
+@dlt.source(name="quickbooks", max_table_nesting=0)
+def quickbooks_source(
+    company_id: str,
+    start_date: pendulum.DateTime,
+    object: str,
+    end_date: pendulum.DateTime | None,
+    client_id: str,
+    client_secret: str,
+    refresh_token: str,
+    environment: str = "production",
+    minor_version: Optional[str] = None,
+) -> Iterable[DltResource]:
+    """Create dlt resources for QuickBooks objects.
+
+    Parameters
+    ----------
+    company_id: str
+        QuickBooks company id (realm id).
+    client_id: str
+        OAuth client id.
+    client_secret: str
+        OAuth client secret.
+    refresh_token: str
+        OAuth refresh token.
+    access_token: Optional[str]
+        Optional access token. If not provided the library will refresh using the
+        provided refresh token.
+    environment: str
+        Either ``"production"`` or ``"sandbox"``.
+    minor_version: Optional[int]
+        QuickBooks API minor version if needed.
+    """
+
+    auth_client = AuthClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        environment=environment,
+        # redirect_uri is not used since we authenticate using refresh token which skips the step of redirect callback.
+        # as redirect_uri is required param, we are passing empty string.
+        redirect_uri="",
+    )
+
+    # https://help.developer.intuit.com/s/article/Validity-of-Refresh-Token
+    client = QuickBooks(
+        auth_client=auth_client,
+        refresh_token=refresh_token,
+        company_id=company_id,
+        minorversion=minor_version,
+    )
+
+    def fetch_object(
+        obj_name: str,
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "lastupdatedtime",
+            initial_value=start_date,  # type: ignore
+            end_value=end_date,  # type: ignore
+            range_start="closed",
+            range_end="closed",
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[List[TDataItem]]:
+        start_pos = 1
+
+        end_dt = updated_at.end_value or pendulum.now(tz="UTC")
+        start_dt = ensure_pendulum_datetime(str(updated_at.last_value)).in_tz("UTC")
+
+        start_str = start_dt.isoformat()
+        end_str = end_dt.isoformat()
+
+        where_clause = f"WHERE MetaData.LastUpdatedTime >= '{start_str}' AND MetaData.LastUpdatedTime < '{end_str}'"
+        while True:
+            query = (
+                f"SELECT * FROM {obj_name} {where_clause} "
+                f"ORDERBY MetaData.LastUpdatedTime ASC STARTPOSITION {start_pos} MAXRESULTS 1000"
+            )
+
+            result = client.query(query)
+
+            items = result.get("QueryResponse", {}).get(obj_name.capitalize(), [])
+            if not items:
+                break
+
+            for item in items:
+                if item.get("MetaData") and item["MetaData"].get("LastUpdatedTime"):
+                    item["lastupdatedtime"] = ensure_pendulum_datetime(
+                        item["MetaData"]["LastUpdatedTime"]
+                    )
+                    item["id"] = item["Id"]
+                    del item["Id"]
+
+                yield item
+
+            if len(items) < 1000:
+                break
+
+            start_pos += 1000
+
+    yield dlt.resource(
+        fetch_object,
+        name=object.lower(),
+        write_disposition="merge",
+        primary_key="id",
+    )(object)

omniload/src/reddit_ads/__init__.py

@@ -0,0 +1,183 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from pendulum import Date
+
+from .helpers import (
+    BASE_URL,
+    LEVEL_ID_FIELDS,
+    RedditAdsAPI,
+    RedditAdsReportAPI,
+)
+
+
+@dlt.source(max_table_nesting=0)
+def reddit_ads_source(
+    access_token: str,
+    account_ids: list[str],
+) -> list[DltResource]:
+    reddit_api = RedditAdsAPI(access_token=access_token)
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def accounts() -> Iterable[TDataItem]:
+        url = f"{BASE_URL}/accounts"
+        account_id_set = set(account_ids)
+        for page in reddit_api.fetch_pages(url):
+            filtered = [item for item in page if str(item.get("id")) in account_id_set]
+            if filtered:
+                yield filtered
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def campaigns() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/campaigns"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def ad_groups() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/ad_groups"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def ads() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/ads"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def posts() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/posts"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def custom_audiences() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/custom_audiences"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def saved_audiences() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/saved_audiences"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def pixels() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/pixels"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    @dlt.resource(write_disposition="replace", primary_key="id")
+    def funding_instruments() -> Iterable[TDataItem]:
+        for account_id in account_ids:
+            url = f"{BASE_URL}/accounts/{account_id}/funding_instruments"
+            for page in reddit_api.fetch_pages(url):
+                for item in page:
+                    item["account_id"] = account_id
+                yield page
+
+    return [
+        accounts,
+        campaigns,
+        ad_groups,
+        ads,
+        posts,
+        custom_audiences,
+        saved_audiences,
+        pixels,
+        funding_instruments,
+    ]
+
+
+@dlt.source(max_table_nesting=0)
+def reddit_ads_analytics_source(
+    access_token: str,
+    account_ids: list[str],
+    level: str,
+    breakdowns: list[str],
+    metrics: list[str],
+    start_date: Date,
+    end_date: Date | None,
+) -> DltResource:
+    level_id_field = LEVEL_ID_FIELDS.get(level, "account_id")
+    primary_key = [level_id_field] + breakdowns
+
+    has_date_breakdown = "date" in breakdowns
+
+    if has_date_breakdown:
+
+        @dlt.resource(write_disposition="merge", primary_key=primary_key)
+        def custom_reports(
+            dateTime=dlt.sources.incremental(
+                "date",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            ),
+        ) -> Iterable[TDataItem]:
+            report_api = RedditAdsReportAPI(
+                access_token=access_token,
+                account_ids=account_ids,
+                level=level,
+                breakdowns=breakdowns,
+                metrics=metrics,
+            )
+
+            actual_end = (
+                dateTime.end_value
+                if dateTime.end_value is not None
+                else pendulum.now().date()
+            )
+
+            yield from report_api.fetch_report(
+                start_date=dateTime.last_value,
+                end_date=actual_end,
+            )
+
+    else:
+
+        @dlt.resource(write_disposition="merge", primary_key=primary_key)
+        def custom_reports() -> Iterable[TDataItem]:
+            report_api = RedditAdsReportAPI(
+                access_token=access_token,
+                account_ids=account_ids,
+                level=level,
+                breakdowns=breakdowns,
+                metrics=metrics,
+            )
+
+            actual_end = end_date if end_date is not None else pendulum.now().date()
+
+            yield from report_api.fetch_report(
+                start_date=start_date,
+                end_date=actual_end,
+            )
+
+    return custom_reports