ingestr 0.13.75__py3-none-any.whl → 0.14.98__py3-none-any.whl

ingestr/src/plusvibeai/settings.py

@@ -0,0 +1,252 @@
+"""PlusVibeAI source settings and constants"""
+
+# Default start date for PlusVibeAI API requests
+DEFAULT_START_DATE = "2020-01-01"
+
+# PlusVibeAI API request timeout in seconds
+REQUEST_TIMEOUT = 300
+
+# Default page size for paginated requests
+DEFAULT_PAGE_SIZE = 100
+
+# Maximum page size (adjust based on API limits)
+MAX_PAGE_SIZE = 1000
+
+# Base API path for PlusVibeAI
+API_BASE_PATH = "/api/v1"
+
+# Campaign fields to retrieve from PlusVibeAI API
+CAMPAIGN_FIELDS = (
+    # Basic Information
+    "id",
+    "camp_name",
+    "parent_camp_id",
+    "campaign_type",
+    "organization_id",
+    "workspace_id",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_lead_sent",
+    "last_paused_at_bounced",
+    # Campaign Configuration
+    "tags",
+    "template_id",
+    "email_accounts",
+    "daily_limit",
+    "interval_limit_in_min",
+    "send_priority",
+    "send_as_txt",
+    # Tracking & Settings
+    "is_emailopened_tracking",
+    "is_unsubscribed_link",
+    "exclude_ooo",
+    "is_acc_based_sending",
+    "send_risky_email",
+    "unsub_blocklist",
+    "other_email_acc",
+    "is_esp_match",
+    "stop_on_lead_replied",
+    # Bounce Settings
+    "is_pause_on_bouncerate",
+    "bounce_rate_limit",
+    "is_paused_at_bounced",
+    # Schedule
+    "schedule",
+    "first_wait_time",
+    "camp_st_date",
+    "camp_end_date",
+    # Events & Sequences
+    "events",
+    "sequences",
+    "sequence_steps",
+    "camp_emails",
+    # Lead Statistics
+    "lead_count",
+    "completed_lead_count",
+    "lead_contacted_count",
+    # Email Performance Metrics
+    "sent_count",
+    "opened_count",
+    "unique_opened_count",
+    "replied_count",
+    "bounced_count",
+    "unsubscribed_count",
+    # Reply Classification
+    "positive_reply_count",
+    "negative_reply_count",
+    "neutral_reply_count",
+    # Daily & Business Metrics
+    "email_sent_today",
+    "opportunity_val",
+    "open_rate",
+    "replied_rate",
+    # Custom Data
+    "custom_fields",
+)
+
+# Lead fields to retrieve from PlusVibeAI API
+LEAD_FIELDS = (
+    # Basic Information
+    "_id",
+    "organization_id",
+    "campaign_id",
+    "workspace_id",
+    # Lead Status & Progress
+    "is_completed",
+    "current_step",
+    "status",
+    "label",
+    # Email Account Info
+    "email_account_id",
+    "email_acc_name",
+    # Campaign Info
+    "camp_name",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_sent_at",
+    # Email Engagement Metrics
+    "sent_step",
+    "replied_count",
+    "opened_count",
+    # Email Verification
+    "is_mx",
+    "mx",
+    # Contact Information
+    "email",
+    "first_name",
+    "last_name",
+    "phone_number",
+    # Address Information
+    "address_line",
+    "city",
+    "state",
+    "country",
+    "country_code",
+    # Professional Information
+    "job_title",
+    "department",
+    "company_name",
+    "company_website",
+    "industry",
+    # Social Media
+    "linkedin_person_url",
+    "linkedin_company_url",
+    # Workflow
+    "total_steps",
+    # Bounce Information
+    "bounce_msg",
+)
+
+# Email Account fields to retrieve from PlusVibeAI API
+EMAIL_ACCOUNT_FIELDS = (
+    # Basic Information
+    "_id",
+    "email",
+    "status",
+    "warmup_status",
+    # Timestamps
+    "timestamp_created",
+    "timestamp_updated",
+    # Payload - nested object containing all configuration
+    "payload",
+    # Payload sub-fields (for reference, stored in payload object):
+    # - name (first_name, last_name)
+    # - warmup (limit, warmup_custom_words, warmup_signature, advanced, increment, reply_rate)
+    # - imap_host, imap_port
+    # - smtp_host, smtp_port
+    # - daily_limit, sending_gap
+    # - reply_to, custom_domain, signature
+    # - tags, cmps
+    # - analytics (health_scores, reply_rates, daily_counters)
+)
+
+# Email fields to retrieve from PlusVibeAI API
+EMAIL_FIELDS = (
+    # Basic Information
+    "id",
+    "message_id",
+    "is_unread",
+    # Lead Information
+    "lead",
+    "lead_id",
+    "campaign_id",
+    # From Address
+    "from_address_email",
+    "from_address_json",
+    # Subject & Content
+    "subject",
+    "content_preview",
+    "body",
+    # Headers & Metadata
+    "headers",
+    "label",
+    "thread_id",
+    "eaccount",
+    # To/CC/BCC Addresses
+    "to_address_email_list",
+    "to_address_json",
+    "cc_address_email_list",
+    "cc_address_json",
+    "bcc_address_email_list",
+    # Timestamps
+    "timestamp_created",
+    "source_modified_at",
+)
+
+# Blocklist fields to retrieve from PlusVibeAI API
+BLOCKLIST_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "value",
+    "created_by_label",
+    # Timestamps
+    "created_at",
+)
+
+# Webhook fields to retrieve from PlusVibeAI API
+WEBHOOK_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "url",
+    "name",
+    "secret",
+    # Configuration
+    "camp_ids",
+    "evt_types",
+    "status",
+    "integration_type",
+    # Settings
+    "ignore_ooo",
+    "ignore_automatic",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_run",
+    # Response Data
+    "last_resp",
+    "last_recv_resp",
+    # User Information
+    "created_by",
+    "modified_by",
+)
+
+# Tag fields to retrieve from PlusVibeAI API
+TAG_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "name",
+    "color",
+    "description",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+)

ingestr/src/revenuecat/__init__.py

@@ -0,0 +1,83 @@
+from typing import Any, Dict, Iterable, Iterator
+
+import aiohttp
+import dlt
+
+from .helpers import (
+    _make_request,
+    _paginate,
+    convert_timestamps_to_iso,
+    create_project_resource,
+    process_customer_with_nested_resources_async,
+)
+
+
+@dlt.source(name="revenuecat", max_table_nesting=0)
+def revenuecat_source(
+    api_key: str,
+    project_id: str = None,
+) -> Iterable[dlt.sources.DltResource]:
+    """
+    RevenueCat source for extracting data from RevenueCat API v2.
+
+    Args:
+        api_key: RevenueCat API v2 secret key with Bearer token format
+        project_id: RevenueCat project ID (required for customers, products, entitlements, offerings, subscriptions, purchases)
+
+    Returns:
+        Iterable of DLT resources for customers, products, entitlements, offerings, purchases, subscriptions, and projects
+    """
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects() -> Iterator[Dict[str, Any]]:
+        """Get list of projects."""
+        # Get projects list
+        data = _make_request(api_key, "/projects")
+        if "items" in data:
+            for project in data["items"]:
+                project = convert_timestamps_to_iso(project, ["created_at"])
+                yield project
+
+    @dlt.resource(
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
+    )
+    def customer_ids():
+        if project_id is None:
+            raise ValueError("project_id is required for customers resource")
+
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
+
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
+
+    # Create project-dependent resources dynamically
+    project_resources = []
+    resource_names = ["products", "entitlements", "offerings"]
+
+    for resource_name in resource_names:
+
+        @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+        def create_resource(resource_name=resource_name) -> Iterator[Dict[str, Any]]:
+            """Get list of project resource."""
+            yield from create_project_resource(resource_name, api_key, project_id)
+
+        # Set the function name for better identification
+        create_resource.__name__ = resource_name
+        project_resources.append(create_resource)
+
+    return [
+        projects,
+        customer_ids,
+        customers,
+        *project_resources,
+    ]

ingestr/src/revenuecat/helpers.py

@@ -0,0 +1,237 @@
+import asyncio
+import time
+from typing import Any, Dict, Iterator, List, Optional
+
+import aiohttp
+import pendulum
+import requests
+
+REVENUECAT_API_BASE = "https://api.revenuecat.com/v2"
+
+
+def _make_request(
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make a REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            response = requests.get(url, headers=headers, params=params or {})
+
+            # Handle rate limiting (429 Too Many Requests)
+            if response.status_code == 429:
+                if attempt < max_retries:
+                    # Wait based on Retry-After header or exponential backoff
+                    retry_after = response.headers.get("Retry-After")
+                    if retry_after:
+                        wait_time = int(retry_after)
+                    else:
+                        wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                    time.sleep(wait_time)
+                    continue
+
+            response.raise_for_status()
+            return response.json()
+
+        except requests.exceptions.RequestException:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                time.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    response.raise_for_status()
+    return response.json()
+
+
+def _paginate(
+    api_key: str, endpoint: str, params: Optional[Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Paginate through RevenueCat API results."""
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = _make_request(api_key, endpoint, current_params)
+
+        if "items" in data and data["items"] is not None:
+            yield data["items"]
+
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+
+def convert_timestamps_to_iso(
+    record: Dict[str, Any], timestamp_fields: List[str]
+) -> Dict[str, Any]:
+    """Convert timestamp fields from milliseconds to ISO format."""
+    for field in timestamp_fields:
+        if field in record and record[field] is not None:
+            timestamp_ms = record[field]
+            dt = pendulum.from_timestamp(timestamp_ms / 1000)
+            record[field] = dt.to_iso8601_string()
+
+    return record
+
+
+async def _make_request_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+    max_retries: int = 3,
+) -> Dict[str, Any]:
+    """Make an async REST API request to RevenueCat API v2 with rate limiting."""
+    auth_header = f"Bearer {api_key}"
+
+    headers = {"Authorization": auth_header, "Content-Type": "application/json"}
+
+    url = f"{REVENUECAT_API_BASE}{endpoint}"
+
+    for attempt in range(max_retries + 1):
+        try:
+            async with session.get(
+                url, headers=headers, params=params or {}
+            ) as response:
+                # Handle rate limiting (429 Too Many Requests)
+                if response.status == 429:
+                    if attempt < max_retries:
+                        # Wait based on Retry-After header or exponential backoff
+                        retry_after = response.headers.get("Retry-After")
+                        if retry_after:
+                            wait_time = int(retry_after)
+                        else:
+                            wait_time = (2**attempt) * 5  # 5, 10, 20 seconds
+
+                        await asyncio.sleep(wait_time)
+                        continue
+
+                response.raise_for_status()
+                return await response.json()
+
+        except aiohttp.ClientError:
+            if attempt < max_retries:
+                wait_time = (2**attempt) * 2  # 2, 4, 8 seconds
+                await asyncio.sleep(wait_time)
+                continue
+            raise
+
+    # If we get here, all retries failed
+    async with session.get(url, headers=headers, params=params or {}) as response:
+        response.raise_for_status()
+        return await response.json()
+
+
+async def _paginate_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    endpoint: str,
+    params: Optional[Dict[str, Any]] = None,
+) -> List[Dict[str, Any]]:
+    """Paginate through RevenueCat API results asynchronously."""
+    items = []
+    current_params = params.copy() if params is not None else {}
+    current_params["limit"] = 1000
+
+    while True:
+        data = await _make_request_async(session, api_key, endpoint, current_params)
+
+        # Collect items from the current page
+        if "items" in data and data["items"] is not None:
+            items.extend(data["items"])
+
+        # Check if there's a next page
+        if "next_page" not in data:
+            break
+
+        # Extract starting_after parameter from next_page URL
+        next_page_url = data["next_page"]
+        if next_page_url and "starting_after=" in next_page_url:
+            starting_after = next_page_url.split("starting_after=")[1].split("&")[0]
+            current_params["starting_after"] = starting_after
+        else:
+            break
+
+    return items
+
+
+async def process_customer_with_nested_resources_async(
+    session: aiohttp.ClientSession,
+    api_key: str,
+    project_id: str,
+    customer: Dict[str, Any],
+) -> Dict[str, Any]:
+    customer_id = customer["id"]
+    customer = convert_timestamps_to_iso(customer, ["first_seen_at", "last_seen_at"])
+    nested_resources = [
+        ("subscriptions", ["purchased_at", "expires_at", "grace_period_expires_at"]),
+        ("purchases", ["purchased_at", "expires_at"]),
+    ]
+
+    async def fetch_and_convert(resource_name, timestamp_fields):
+        if resource_name not in customer or customer[resource_name] is None:
+            endpoint = f"/projects/{project_id}/customers/{customer_id}/{resource_name}"
+            customer[resource_name] = await _paginate_async(session, api_key, endpoint)
+        if (
+            timestamp_fields
+            and resource_name in customer
+            and customer[resource_name] is not None
+        ):
+            for item in customer[resource_name]:
+                convert_timestamps_to_iso(item, timestamp_fields)
+
+    await asyncio.gather(
+        *[
+            fetch_and_convert(resource_name, timestamp_fields)
+            for resource_name, timestamp_fields in nested_resources
+        ]
+    )
+
+    return customer
+
+
+def create_project_resource(
+    resource_name: str,
+    api_key: str,
+    project_id: str = None,
+    timestamp_fields: List[str] = None,
+) -> Iterator[Dict[str, Any]]:
+    """
+    Helper function to create DLT resources for project-dependent endpoints.
+
+    Args:
+        resource_name: Name of the resource (e.g., 'products', 'entitlements', 'offerings')
+        api_key: RevenueCat API key
+        project_id: RevenueCat project ID
+        timestamp_fields: List of timestamp fields to convert to ISO format
+
+    Returns:
+        Iterator of resource data
+    """
+    if project_id is None:
+        raise ValueError(f"project_id is required for {resource_name} resource")
+
+    endpoint = f"/projects/{project_id}/{resource_name}"
+    default_timestamp_fields = timestamp_fields or ["created_at", "updated_at"]
+
+    for item in _paginate(api_key, endpoint):
+        item = convert_timestamps_to_iso(item, default_timestamp_fields)
+        yield item

ingestr/src/salesforce/__init__.py

@@ -13,6 +13,8 @@ def salesforce_source(
     username: str,
     password: str,
     token: str,
+    domain: str,
+    custom_object: str = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from Salesforce using the Salesforce API.
@@ -26,7 +28,7 @@ def salesforce_source(
         DltResource: Data resources from Salesforce.
     """
 
-    client = Salesforce(username, password, token)
+    client = Salesforce(username, password, token, domain=domain)
 
     # define resources
     @dlt.resource(write_disposition="replace")
@@ -37,7 +39,7 @@ def salesforce_source(
     def user_role() -> Iterable[TDataItem]:
         yield get_records(client, "UserRole")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -47,7 +49,7 @@ def salesforce_source(
             client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_line_item(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -57,7 +59,7 @@ def salesforce_source(
             client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def opportunity_contact_role(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -70,7 +72,7 @@ def salesforce_source(
             "SystemModstamp",
         )
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def account(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "LastModifiedDate", initial_value=None
@@ -92,7 +94,7 @@ def salesforce_source(
     def campaign() -> Iterable[TDataItem]:
         yield get_records(client, "Campaign")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def campaign_member(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -114,7 +116,7 @@ def salesforce_source(
     def pricebook_entry() -> Iterable[TDataItem]:
         yield get_records(client, "PricebookEntry")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def task(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -122,7 +124,7 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
 
-    @dlt.resource(write_disposition="merge")
+    @dlt.resource(write_disposition="merge", primary_key="id")
     def event(
         last_timestamp: incremental[str] = dlt.sources.incremental(
             "SystemModstamp", initial_value=None
@@ -130,6 +132,10 @@ def salesforce_source(
     ) -> Iterable[TDataItem]:
         yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
 
+    @dlt.resource(write_disposition="replace")
+    def custom() -> Iterable[TDataItem]:
+        yield get_records(client, custom_object)
+
     return (
         user,
         user_role,
@@ -146,4 +152,5 @@ def salesforce_source(
         pricebook_entry,
         task,
         event,
+        custom,
     )

ingestr/src/shopify/__init__.py

@@ -669,7 +669,7 @@ def shopify_source(
             params["updated_at_max"] = updated_at.end_value.isoformat()
         yield from client.get_pages("customers", params)
 
-    @dlt.resource(primary_key="id", write_disposition="append")
+    @dlt.resource(primary_key="id", write_disposition="merge")
     def events(
         created_at: dlt.sources.incremental[
             pendulum.DateTime