ingestr 0.14.3__py3-none-any.whl → 0.14.5__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.3"
+version = "v0.14.5"

ingestr/src/factory.py

@@ -70,6 +70,7 @@ from ingestr.src.sources import (
     PhantombusterSource,
     PinterestSource,
     PipedriveSource,
+    PlusVibeAISource,
     QuickBooksSource,
     RevenueCatSource,
     S3Source,
@@ -212,6 +213,7 @@ class SourceDestinationFactory:
         "clickup": ClickupSource,
         "influxdb": InfluxDBSource,
         "wise": WiseSource,
+        "plusvibeai": PlusVibeAISource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/freshdesk/__init__.py

@@ -20,6 +20,7 @@ def freshdesk_source(
     end_date: Optional[pendulum.DateTime] = None,
     per_page: int = 100,
     endpoints: Optional[List[str]] = None,
+    query: Optional[str] = None,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from specified Freshdesk API endpoints.
@@ -72,6 +73,7 @@ def freshdesk_source(
             per_page=per_page,
             start_date=start_date,
             end_date=end_date,
+            query=query,
         )
 
     # Set default endpoints if not provided

ingestr/src/freshdesk/freshdesk_client.py

@@ -2,7 +2,7 @@
 
 import logging
 import time
-from typing import Any, Dict, Iterable
+from typing import Any, Dict, Iterable, Optional
 
 import pendulum
 from dlt.common.typing import TDataItem
@@ -70,6 +70,7 @@ class FreshdeskClient:
         per_page: int,
         start_date: pendulum.DateTime,
         end_date: pendulum.DateTime,
+        query: Optional[str] = None,
     ) -> Iterable[TDataItem]:
         """
         Fetches a paginated response from a specified endpoint.
@@ -79,6 +80,9 @@ class FreshdeskClient:
         updated at the specified timestamp.
         """
         page = 1
+        if query is not None:
+            query = query.replace('"', "").strip()
+
         while True:
             # Construct the URL for the specific endpoint
             url = f"{self.base_url}/{endpoint}"
@@ -93,11 +97,21 @@ class FreshdeskClient:
 
                 params[param_key] = start_date.to_iso8601_string()
 
+            if query and endpoint == "tickets":
+                url = f"{self.base_url}/search/tickets"
+                params = {
+                    "query": f'"{query}"',
+                    "page": page,
+                }
+
             # Handle requests with rate-limiting
             # A maximum of 300 pages (30000 tickets) will be returned.
             response = self._request_with_rate_limit(url, params=params)
             data = response.json()
 
+            if query and endpoint == "tickets":
+                data = data["results"]
+
             if not data:
                 break  # Stop if no data or max page limit reached
 

ingestr/src/jira_source/__init__.py

@@ -37,6 +37,7 @@ def jira_source() -> Any:
         resolutions,
         project_versions,
         project_components,
+        events,
     ]
 
 
@@ -65,7 +66,11 @@ def projects(
     yield from client.get_projects(expand=expand, recent=recent)
 
 
-@dlt.resource(write_disposition="merge", primary_key="id")
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="id",
+    max_table_nesting=2,
+)
 def issues(
     base_url: str = dlt.secrets.value,
     email: str = dlt.secrets.value,
@@ -312,3 +317,24 @@ def project_components(
         return []
 
     return list(client.get_project_components(project_key))
+
+
+@dlt.resource(write_disposition="replace")
+def events(
+    base_url: str = dlt.secrets.value,
+    email: str = dlt.secrets.value,
+    api_token: str = dlt.secrets.value,
+) -> Iterable[TDataItem]:
+    """
+    Fetches all event types from Jira (e.g., Issue Created, Issue Updated, etc.).
+
+    Args:
+        base_url (str): Jira instance URL
+        email (str): User email for authentication
+        api_token (str): API token for authentication
+
+    Yields:
+        dict: The event data.
+    """
+    client = get_client(base_url, email, api_token)
+    yield from client.get_events()

ingestr/src/jira_source/helpers.py

@@ -98,8 +98,6 @@ class JiraClient:
 
         for attempt in range(max_retries + 1):
             try:
-                logger.debug(f"Making request to {url} (attempt {attempt + 1})")
-
                 response = requests.request(
                     method=method,
                     url=url,
@@ -214,10 +212,6 @@ class JiraClient:
         consecutive_empty_pages = 0
         max_empty_pages = 3
 
-        logger.info(
-            f"Starting paginated request to {endpoint} with page_size={page_size}"
-        )
-
         while True:
             try:
                 response = self._make_request(endpoint, params)
@@ -238,7 +232,6 @@ class JiraClient:
                     is_last = True
                 else:
                     # Single item response
-                    logger.debug(f"Received single item response from {endpoint}")
                     yield response
                     break
 
@@ -253,27 +246,18 @@ class JiraClient:
                 else:
                     consecutive_empty_pages = 0
 
-                logger.debug(
-                    f"Retrieved {len(items)} items from {endpoint} (page {params['startAt'] // page_size + 1})"
-                )
-
                 for item in items:
                     if max_results and total_returned >= max_results:
-                        logger.info(f"Reached max_results limit of {max_results}")
                         return
                     yield item
                     total_returned += 1
 
                 # Check if we've reached the end
                 if is_last or len(items) < page_size:
-                    logger.debug(f"Reached end of pagination for {endpoint}")
                     break
 
                 # Check if we've got all available items
                 if total and total_returned >= total:
-                    logger.debug(
-                        f"Retrieved all {total} available items from {endpoint}"
-                    )
                     break
 
                 # Move to next page
@@ -295,10 +279,6 @@ class JiraClient:
                 )
                 raise JiraAPIError(f"Pagination failed: {str(e)}")
 
-        logger.info(
-            f"Completed pagination for {endpoint}, returned {total_returned} items"
-        )
-
     def search_issues(
         self,
         jql: str,
@@ -327,7 +307,7 @@ class JiraClient:
             params["expand"] = expand
 
         yield from self.get_paginated(
-            "search", params=params, page_size=page_size, max_results=max_results
+            "search/jql", params=params, page_size=page_size, max_results=max_results
         )
 
     def get_projects(
@@ -433,6 +413,13 @@ class JiraClient:
         """
         yield from self.get_paginated(f"project/{project_key}/component")
 
+    def get_events(self) -> Iterator[Dict[str, Any]]:
+        """Get all events (issue events like created, updated, etc.)."""
+        response = self._make_request("events")
+        if isinstance(response, list):
+            for event in response:
+                yield event
+
 
 def get_client(
     base_url: str, email: str, api_token: str, timeout: int = REQUEST_TIMEOUT

ingestr/src/plusvibeai/__init__.py

@@ -0,0 +1,335 @@
+"""
+This source provides data extraction from PlusVibeAI via the REST API.
+
+It defines functions to fetch data from different parts of PlusVibeAI including
+campaigns and other marketing analytics data.
+"""
+
+from typing import Any, Iterable, Optional
+
+import dlt
+from dlt.common.typing import TDataItem
+
+from .helpers import get_client
+from .settings import DEFAULT_PAGE_SIZE, DEFAULT_START_DATE
+
+
+@dlt.source
+def plusvibeai_source() -> Any:
+    """
+    The main function that runs all the other functions to fetch data from PlusVibeAI.
+
+    Returns:
+        Sequence[DltResource]: A sequence of DltResource objects containing the fetched data.
+    """
+    return [
+        campaigns,
+        leads,
+        email_accounts,
+        emails,
+        blocklist,
+        webhooks,
+        tags,
+    ]
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="id",
+    max_table_nesting=0,  # Keep nested objects (schedule, sequences) as JSON columns
+)
+def campaigns(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "modified_at",  # PlusVibeAI uses modified_at for updates
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches campaigns from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication (get from https://app.plusvibe.ai/v2/settings/api-access/)
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch updated campaigns
+
+    Yields:
+        dict: The campaign data with nested objects (schedule, sequences, etc.) as JSON.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for campaign in client.get_campaigns(
+        page_size=DEFAULT_PAGE_SIZE, max_results=max_results
+    ):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            campaign_updated = campaign.get("modified_at")
+            if campaign_updated and campaign_updated < updated.start_value:
+                continue
+
+        yield campaign
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="_id",
+    max_table_nesting=0,
+)
+def leads(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches leads from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch updated leads
+
+    Yields:
+        dict: The lead data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for lead in client.get_leads(page_size=DEFAULT_PAGE_SIZE, max_results=max_results):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            lead_updated = lead.get("modified_at")
+            if lead_updated and lead_updated < updated.start_value:
+                continue
+
+        yield lead
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="_id",
+    max_table_nesting=0,
+)
+def email_accounts(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "timestamp_updated",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches email accounts from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch updated email accounts
+
+    Yields:
+        dict: The email account data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for account in client.get_email_accounts(
+        page_size=DEFAULT_PAGE_SIZE, max_results=max_results
+    ):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            account_updated = account.get("timestamp_updated")
+            if account_updated and account_updated < updated.start_value:
+                continue
+
+        yield account
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="id",
+    max_table_nesting=0,
+)
+def emails(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "timestamp_created",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches emails from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch emails
+
+    Yields:
+        dict: The email data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for email in client.get_emails(max_results=max_results):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            email_created = email.get("timestamp_created")
+            if email_created and email_created < updated.start_value:
+                continue
+
+        yield email
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="_id",
+    max_table_nesting=0,
+)
+def blocklist(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "created_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches blocklist entries from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch blocklist entries
+
+    Yields:
+        dict: The blocklist entry data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for entry in client.get_blocklist(
+        page_size=DEFAULT_PAGE_SIZE, max_results=max_results
+    ):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            entry_created = entry.get("created_at")
+            if entry_created and entry_created < updated.start_value:
+                continue
+
+        yield entry
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="_id",
+    max_table_nesting=0,
+)
+def webhooks(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches webhooks from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch updated webhooks
+
+    Yields:
+        dict: The webhook data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for webhook in client.get_webhooks(
+        page_size=DEFAULT_PAGE_SIZE, max_results=max_results
+    ):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            webhook_updated = webhook.get("modified_at")
+            if webhook_updated and webhook_updated < updated.start_value:
+                continue
+
+        yield webhook
+
+
+@dlt.resource(
+    write_disposition="merge",
+    primary_key="_id",
+    max_table_nesting=0,
+)
+def tags(
+    api_key: str = dlt.secrets.value,
+    workspace_id: str = dlt.secrets.value,
+    base_url: str = "https://api.plusvibe.ai",
+    max_results: Optional[int] = None,
+    updated: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "modified_at",
+        initial_value=DEFAULT_START_DATE,
+        range_end="closed",
+        range_start="closed",
+    ),
+) -> Iterable[TDataItem]:
+    """
+    Fetches tags from PlusVibeAI.
+
+    Args:
+        api_key (str): API key for authentication
+        workspace_id (str): Workspace ID to access
+        base_url (str): PlusVibeAI API base URL
+        max_results (int): Maximum number of results to return
+        updated (str): The date from which to fetch updated tags
+
+    Yields:
+        dict: The tag data.
+    """
+    client = get_client(api_key, workspace_id, base_url)
+
+    for tag in client.get_tags(page_size=DEFAULT_PAGE_SIZE, max_results=max_results):
+        # Apply incremental filter if needed
+        if updated.start_value:
+            tag_updated = tag.get("modified_at")
+            if tag_updated and tag_updated < updated.start_value:
+                continue
+
+        yield tag