ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/plusvibeai/settings.py

@@ -0,0 +1,252 @@
+"""PlusVibeAI source settings and constants"""
+
+# Default start date for PlusVibeAI API requests
+DEFAULT_START_DATE = "2020-01-01"
+
+# PlusVibeAI API request timeout in seconds
+REQUEST_TIMEOUT = 300
+
+# Default page size for paginated requests
+DEFAULT_PAGE_SIZE = 100
+
+# Maximum page size (adjust based on API limits)
+MAX_PAGE_SIZE = 1000
+
+# Base API path for PlusVibeAI
+API_BASE_PATH = "/api/v1"
+
+# Campaign fields to retrieve from PlusVibeAI API
+CAMPAIGN_FIELDS = (
+    # Basic Information
+    "id",
+    "camp_name",
+    "parent_camp_id",
+    "campaign_type",
+    "organization_id",
+    "workspace_id",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_lead_sent",
+    "last_paused_at_bounced",
+    # Campaign Configuration
+    "tags",
+    "template_id",
+    "email_accounts",
+    "daily_limit",
+    "interval_limit_in_min",
+    "send_priority",
+    "send_as_txt",
+    # Tracking & Settings
+    "is_emailopened_tracking",
+    "is_unsubscribed_link",
+    "exclude_ooo",
+    "is_acc_based_sending",
+    "send_risky_email",
+    "unsub_blocklist",
+    "other_email_acc",
+    "is_esp_match",
+    "stop_on_lead_replied",
+    # Bounce Settings
+    "is_pause_on_bouncerate",
+    "bounce_rate_limit",
+    "is_paused_at_bounced",
+    # Schedule
+    "schedule",
+    "first_wait_time",
+    "camp_st_date",
+    "camp_end_date",
+    # Events & Sequences
+    "events",
+    "sequences",
+    "sequence_steps",
+    "camp_emails",
+    # Lead Statistics
+    "lead_count",
+    "completed_lead_count",
+    "lead_contacted_count",
+    # Email Performance Metrics
+    "sent_count",
+    "opened_count",
+    "unique_opened_count",
+    "replied_count",
+    "bounced_count",
+    "unsubscribed_count",
+    # Reply Classification
+    "positive_reply_count",
+    "negative_reply_count",
+    "neutral_reply_count",
+    # Daily & Business Metrics
+    "email_sent_today",
+    "opportunity_val",
+    "open_rate",
+    "replied_rate",
+    # Custom Data
+    "custom_fields",
+)
+
+# Lead fields to retrieve from PlusVibeAI API
+LEAD_FIELDS = (
+    # Basic Information
+    "_id",
+    "organization_id",
+    "campaign_id",
+    "workspace_id",
+    # Lead Status & Progress
+    "is_completed",
+    "current_step",
+    "status",
+    "label",
+    # Email Account Info
+    "email_account_id",
+    "email_acc_name",
+    # Campaign Info
+    "camp_name",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_sent_at",
+    # Email Engagement Metrics
+    "sent_step",
+    "replied_count",
+    "opened_count",
+    # Email Verification
+    "is_mx",
+    "mx",
+    # Contact Information
+    "email",
+    "first_name",
+    "last_name",
+    "phone_number",
+    # Address Information
+    "address_line",
+    "city",
+    "state",
+    "country",
+    "country_code",
+    # Professional Information
+    "job_title",
+    "department",
+    "company_name",
+    "company_website",
+    "industry",
+    # Social Media
+    "linkedin_person_url",
+    "linkedin_company_url",
+    # Workflow
+    "total_steps",
+    # Bounce Information
+    "bounce_msg",
+)
+
+# Email Account fields to retrieve from PlusVibeAI API
+EMAIL_ACCOUNT_FIELDS = (
+    # Basic Information
+    "_id",
+    "email",
+    "status",
+    "warmup_status",
+    # Timestamps
+    "timestamp_created",
+    "timestamp_updated",
+    # Payload - nested object containing all configuration
+    "payload",
+    # Payload sub-fields (for reference, stored in payload object):
+    # - name (first_name, last_name)
+    # - warmup (limit, warmup_custom_words, warmup_signature, advanced, increment, reply_rate)
+    # - imap_host, imap_port
+    # - smtp_host, smtp_port
+    # - daily_limit, sending_gap
+    # - reply_to, custom_domain, signature
+    # - tags, cmps
+    # - analytics (health_scores, reply_rates, daily_counters)
+)
+
+# Email fields to retrieve from PlusVibeAI API
+EMAIL_FIELDS = (
+    # Basic Information
+    "id",
+    "message_id",
+    "is_unread",
+    # Lead Information
+    "lead",
+    "lead_id",
+    "campaign_id",
+    # From Address
+    "from_address_email",
+    "from_address_json",
+    # Subject & Content
+    "subject",
+    "content_preview",
+    "body",
+    # Headers & Metadata
+    "headers",
+    "label",
+    "thread_id",
+    "eaccount",
+    # To/CC/BCC Addresses
+    "to_address_email_list",
+    "to_address_json",
+    "cc_address_email_list",
+    "cc_address_json",
+    "bcc_address_email_list",
+    # Timestamps
+    "timestamp_created",
+    "source_modified_at",
+)
+
+# Blocklist fields to retrieve from PlusVibeAI API
+BLOCKLIST_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "value",
+    "created_by_label",
+    # Timestamps
+    "created_at",
+)
+
+# Webhook fields to retrieve from PlusVibeAI API
+WEBHOOK_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "url",
+    "name",
+    "secret",
+    # Configuration
+    "camp_ids",
+    "evt_types",
+    "status",
+    "integration_type",
+    # Settings
+    "ignore_ooo",
+    "ignore_automatic",
+    # Timestamps
+    "created_at",
+    "modified_at",
+    "last_run",
+    # Response Data
+    "last_resp",
+    "last_recv_resp",
+    # User Information
+    "created_by",
+    "modified_by",
+)
+
+# Tag fields to retrieve from PlusVibeAI API
+TAG_FIELDS = (
+    # Basic Information
+    "_id",
+    "workspace_id",
+    "org_id",
+    "name",
+    "color",
+    "description",
+    "status",
+    # Timestamps
+    "created_at",
+    "modified_at",
+)

ingestr/src/quickbooks/__init__.py

@@ -0,0 +1,117 @@
+"""QuickBooks source built on top of python-quickbooks."""
+
+from typing import Iterable, Iterator, List, Optional
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from intuitlib.client import AuthClient  # type: ignore
+
+from quickbooks import QuickBooks  # type: ignore
+
+
+@dlt.source(name="quickbooks", max_table_nesting=0)
+def quickbooks_source(
+    company_id: str,
+    start_date: pendulum.DateTime,
+    object: str,
+    end_date: pendulum.DateTime | None,
+    client_id: str,
+    client_secret: str,
+    refresh_token: str,
+    environment: str = "production",
+    minor_version: Optional[str] = None,
+) -> Iterable[DltResource]:
+    """Create dlt resources for QuickBooks objects.
+
+    Parameters
+    ----------
+    company_id: str
+        QuickBooks company id (realm id).
+    client_id: str
+        OAuth client id.
+    client_secret: str
+        OAuth client secret.
+    refresh_token: str
+        OAuth refresh token.
+    access_token: Optional[str]
+        Optional access token. If not provided the library will refresh using the
+        provided refresh token.
+    environment: str
+        Either ``"production"`` or ``"sandbox"``.
+    minor_version: Optional[int]
+        QuickBooks API minor version if needed.
+    """
+
+    auth_client = AuthClient(
+        client_id=client_id,
+        client_secret=client_secret,
+        environment=environment,
+        # redirect_uri is not used since we authenticate using refresh token which skips the step of redirect callback.
+        # as redirect_uri is required param, we are passing empty string.
+        redirect_uri="",
+    )
+
+    # https://help.developer.intuit.com/s/article/Validity-of-Refresh-Token
+    client = QuickBooks(
+        auth_client=auth_client,
+        refresh_token=refresh_token,
+        company_id=company_id,
+        minorversion=minor_version,
+    )
+
+    def fetch_object(
+        obj_name: str,
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "lastupdatedtime",
+            initial_value=start_date,  # type: ignore
+            end_value=end_date,  # type: ignore
+            range_start="closed",
+            range_end="closed",
+            allow_external_schedulers=True,
+        ),
+    ) -> Iterator[List[TDataItem]]:
+        start_pos = 1
+
+        end_dt = updated_at.end_value or pendulum.now(tz="UTC")
+        start_dt = ensure_pendulum_datetime(str(updated_at.last_value)).in_tz("UTC")
+
+        start_str = start_dt.isoformat()
+        end_str = end_dt.isoformat()
+
+        where_clause = f"WHERE MetaData.LastUpdatedTime >= '{start_str}' AND MetaData.LastUpdatedTime < '{end_str}'"
+        while True:
+            query = (
+                f"SELECT * FROM {obj_name} {where_clause} "
+                f"ORDERBY MetaData.LastUpdatedTime ASC STARTPOSITION {start_pos} MAXRESULTS 1000"
+            )
+
+            result = client.query(query)
+
+            items = result.get("QueryResponse", {}).get(obj_name.capitalize(), [])
+            if not items:
+                break
+
+            for item in items:
+                if item.get("MetaData") and item["MetaData"].get("LastUpdatedTime"):
+                    item["lastupdatedtime"] = ensure_pendulum_datetime(
+                        item["MetaData"]["LastUpdatedTime"]
+                    )
+                    item["id"] = item["Id"]
+                    del item["Id"]
+
+                yield item
+
+            if len(items) < 1000:
+                break
+
+            start_pos += 1000
+
+    yield dlt.resource(
+        fetch_object,
+        name=object.lower(),
+        write_disposition="merge",
+        primary_key="id",
+    )(object)

ingestr/src/resource.py

@@ -0,0 +1,40 @@
+from typing import Callable
+
+from dlt.sources import DltResource, DltSource
+
+
+def for_each(
+    source: DltSource | DltResource, ex: Callable[[DltResource], None | DltResource]
+):
+    """
+    Apply a function to each resource in a source.
+    """
+    if hasattr(source, "selected_resources") and source.selected_resources:
+        resource_names = list(source.selected_resources.keys())
+        for res in resource_names:
+            ex(source.resources[res])  # type: ignore[union-attr]
+    else:
+        ex(source)  # type: ignore[arg-type]
+
+
+class TypeHintMap:
+    def __init__(self):
+        self.handled_typehints = False
+
+    def type_hint_map(self, item):
+        if self.handled_typehints:
+            return item
+
+        array_cols = []
+        for col in item:
+            if isinstance(item[col], (list, tuple)):
+                array_cols.append(col)
+        if array_cols:
+            import dlt
+
+            source = dlt.current.source()
+            columns = [{"name": col, "data_type": "json"} for col in array_cols]
+            for_each(source, lambda x: x.apply_hints(columns=columns))
+
+        self.handled_typehints = True
+        return item

ingestr/src/revenuecat/__init__.py

@@ -0,0 +1,83 @@
+from typing import Any, Dict, Iterable, Iterator
+
+import aiohttp
+import dlt
+
+from .helpers import (
+    _make_request,
+    _paginate,
+    convert_timestamps_to_iso,
+    create_project_resource,
+    process_customer_with_nested_resources_async,
+)
+
+
+@dlt.source(name="revenuecat", max_table_nesting=0)
+def revenuecat_source(
+    api_key: str,
+    project_id: str = None,
+) -> Iterable[dlt.sources.DltResource]:
+    """
+    RevenueCat source for extracting data from RevenueCat API v2.
+
+    Args:
+        api_key: RevenueCat API v2 secret key with Bearer token format
+        project_id: RevenueCat project ID (required for customers, products, entitlements, offerings, subscriptions, purchases)
+
+    Returns:
+        Iterable of DLT resources for customers, products, entitlements, offerings, purchases, subscriptions, and projects
+    """
+
+    @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
+    def projects() -> Iterator[Dict[str, Any]]:
+        """Get list of projects."""
+        # Get projects list
+        data = _make_request(api_key, "/projects")
+        if "items" in data:
+            for project in data["items"]:
+                project = convert_timestamps_to_iso(project, ["created_at"])
+                yield project
+
+    @dlt.resource(
+        name="customer_ids",
+        write_disposition="replace",
+        selected=False,
+        parallelized=True,
+    )
+    def customer_ids():
+        if project_id is None:
+            raise ValueError("project_id is required for customers resource")
+
+        yield _paginate(api_key, f"/projects/{project_id}/customers")
+
+    @dlt.transformer(
+        data_from=customer_ids, write_disposition="replace", parallelized=True
+    )
+    async def customers(customers) -> Iterator[Dict[str, Any]]:
+        async with aiohttp.ClientSession() as session:
+            for customer in customers:
+                yield await process_customer_with_nested_resources_async(
+                    session, api_key, project_id, customer
+                )
+
+    # Create project-dependent resources dynamically
+    project_resources = []
+    resource_names = ["products", "entitlements", "offerings"]
+
+    for resource_name in resource_names:
+
+        @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+        def create_resource(resource_name=resource_name) -> Iterator[Dict[str, Any]]:
+            """Get list of project resource."""
+            yield from create_project_resource(resource_name, api_key, project_id)
+
+        # Set the function name for better identification
+        create_resource.__name__ = resource_name
+        project_resources.append(create_resource)
+
+    return [
+        projects,
+        customer_ids,
+        customers,
+        *project_resources,
+    ]