ingestr 0.14.0__py3-none-any.whl → 0.14.2__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.0"
+version = "v0.14.2"

ingestr/src/factory.py

@@ -54,6 +54,7 @@ from ingestr.src.sources import (
     GorgiasSource,
     HubspotSource,
     InfluxDBSource,
+    IntercomSource,
     IsocPulseSource,
     KafkaSource,
     KinesisSource,
@@ -166,6 +167,7 @@ class SourceDestinationFactory:
         "fluxx": FluxxSource,
         "slack": SlackSource,
         "hubspot": HubspotSource,
+        "intercom": IntercomSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
         "mixpanel": MixpanelSource,

ingestr/src/fluxx/__init__.py

@@ -9879,10 +9879,7 @@ def fluxx_source(
                 if field_name in fields_to_extract:
                     filtered_fields[field_name] = fields_to_extract[field_name]
                 else:
-                    # If field is not in the resource configuration, log a warning but continue
-                    print(
-                        f"Warning: Field '{field_name}' not found in {resource_name} configuration"
-                    )
+                    filtered_fields[field_name] = {}
 
             fields_to_extract = filtered_fields
 

ingestr/src/fluxx/helpers.py

@@ -43,12 +43,6 @@ def fluxx_api_request(
         "Authorization": f"Bearer {access_token}",
         "Content-Type": "application/json",
     }
-    # print(f"Making request to Fluxx API:")
-    # print(f"  Method: {method}")
-    # print(f"  URL: {url}")
-    # print(f"  Headers: {headers}")
-    # print(f"  Params: {params}")
-    # print(f"  Data: {data}")
 
     response = requests.request(
         method=method,
@@ -88,7 +82,6 @@ def paginate_fluxx_resource(
             params=params,
         )
 
-        print("resssponse", response)
         if not response:
             break
 

ingestr/src/intercom/__init__.py

@@ -0,0 +1,142 @@
+"""
+Intercom source implementation for data ingestion.
+
+This module provides DLT sources for retrieving data from Intercom API endpoints
+including contacts, companies, conversations, tickets, and more.
+"""
+
+from typing import Optional, Sequence
+
+import dlt
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TAnyDateTime
+from dlt.sources import DltResource, DltSource
+
+from .helpers import (
+    IntercomAPIClient,
+    IntercomCredentialsAccessToken,
+    TIntercomCredentials,
+    convert_datetime_to_timestamp,
+    create_resource_from_config,
+    transform_company,
+    transform_contact,
+    transform_conversation,
+)
+from .helpers import (
+    IntercomCredentialsOAuth as IntercomCredentialsOAuth,
+)
+from .settings import (
+    DEFAULT_START_DATE,
+    RESOURCE_CONFIGS,
+)
+
+
+@dlt.source(name="intercom", max_table_nesting=0)
+def intercom_source(
+    credentials: TIntercomCredentials = dlt.secrets.value,
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> Sequence[DltResource]:
+    """
+    A DLT source that retrieves data from Intercom API.
+
+    This source provides access to various Intercom resources including contacts,
+    companies, conversations, tickets, and more. It supports incremental loading
+    for resources that track updated timestamps.
+
+    Args:
+        credentials: Intercom API credentials (AccessToken or OAuth).
+            Defaults to dlt.secrets.value.
+        start_date: The start date for incremental loading.
+            Defaults to January 1, 2020.
+        end_date: Optional end date for incremental loading.
+            If not provided, loads all data from start_date to present.
+
+    Returns:
+        Sequence of DLT resources for different Intercom endpoints.
+
+    Example:
+        >>> source = intercom_source(
+        ...     credentials=IntercomCredentialsAccessToken(
+        ...         access_token="your_token",
+        ...         region="us"
+        ...     ),
+        ...     start_date=datetime(2024, 1, 1)
+        ... )
+    """
+    # Initialize API client
+    api_client = IntercomAPIClient(credentials)
+
+    # Convert dates to pendulum and then to unix timestamps for Intercom API
+    start_date_obj = ensure_pendulum_datetime(start_date) if start_date else None
+    end_date_obj = ensure_pendulum_datetime(end_date) if end_date else None
+
+    # Convert to unix timestamps for API compatibility
+    # Use default start date if none provided
+    if not start_date_obj:
+        from .settings import DEFAULT_START_DATE
+
+        start_date_obj = ensure_pendulum_datetime(DEFAULT_START_DATE)
+
+    start_timestamp = convert_datetime_to_timestamp(start_date_obj)
+    end_timestamp = (
+        convert_datetime_to_timestamp(end_date_obj) if end_date_obj else None
+    )
+
+    # Transform function mapping
+    transform_functions = {
+        "transform_contact": transform_contact,
+        "transform_company": transform_company,
+        "transform_conversation": transform_conversation,
+    }
+
+    # Generate all resources from configuration
+    resources = []
+    for resource_name, config in RESOURCE_CONFIGS.items():
+        resource_func = create_resource_from_config(
+            resource_name,
+            config,
+            api_client,
+            start_timestamp,
+            end_timestamp,
+            transform_functions,
+        )
+
+        # Call the resource function to get the actual resource
+        resources.append(resource_func())
+
+    return resources
+
+
+def intercom(
+    api_key: str,
+    region: str = "us",
+    start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
+    end_date: Optional[TAnyDateTime] = None,
+) -> DltSource:
+    """
+    Convenience function to create Intercom source with access token.
+
+    Args:
+        api_key: Intercom API access token.
+        region: Data region (us, eu, or au). Defaults to "us".
+        start_date: Start date for incremental loading.
+        end_date: Optional end date for incremental loading.
+
+    Returns:
+        Sequence of DLT resources.
+
+    Example:
+        >>> source = intercom(
+        ...     api_key="your_access_token",
+        ...     region="us",
+        ...     start_date=datetime(2024, 1, 1)
+        ... )
+    """
+    credentials = IntercomCredentialsAccessToken(access_token=api_key, region=region)
+
+    return intercom_source(
+        credentials=credentials,
+        start_date=start_date,
+        end_date=end_date,
+    )

ingestr/src/intercom/helpers.py

@@ -0,0 +1,674 @@
+"""
+Helper functions and API client for Intercom integration.
+"""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Callable, Dict, Iterator, Optional, Union
+
+from dlt.common.typing import TDataItem, TDataItems, TSecretValue
+
+from ingestr.src.http_client import create_client
+
+from .settings import (
+    API_VERSION,
+    DEFAULT_PAGE_SIZE,
+    REGIONAL_ENDPOINTS,
+)
+
+
+class PaginationType(Enum):
+    """Types of pagination supported by Intercom API."""
+
+    CURSOR = "cursor"
+    SCROLL = "scroll"
+    SIMPLE = "simple"  # No pagination, single page
+    SEARCH = "search"  # Search API pagination
+
+
+class IntercomCredentials:
+    """Base class for Intercom credentials."""
+
+    def __init__(self, region: str = "us"):
+        self.region = region
+        if self.region not in REGIONAL_ENDPOINTS:
+            raise ValueError(
+                f"Invalid region: {self.region}. Must be one of {list(REGIONAL_ENDPOINTS.keys())}"
+            )
+
+    @property
+    def base_url(self) -> str:
+        """Get the base URL for the specified region."""
+        return REGIONAL_ENDPOINTS[self.region]
+
+
+@dataclass
+class IntercomCredentialsAccessToken(IntercomCredentials):
+    """Credentials for Intercom API using Access Token authentication."""
+
+    access_token: TSecretValue
+    region: str = "us"  # us, eu, or au
+
+    def __post_init__(self):
+        super().__init__(self.region)
+
+
+@dataclass
+class IntercomCredentialsOAuth(IntercomCredentials):
+    """Credentials for Intercom API using OAuth authentication."""
+
+    oauth_token: TSecretValue
+    region: str = "us"  # us, eu, or au
+
+    def __post_init__(self):
+        super().__init__(self.region)
+
+
+TIntercomCredentials = Union[IntercomCredentialsAccessToken, IntercomCredentialsOAuth]
+
+
+class IntercomAPIClient:
+    """
+    API client for making requests to Intercom API.
+    Handles authentication, pagination, and rate limiting.
+    """
+
+    def __init__(self, credentials: TIntercomCredentials):
+        """
+        Initialize the Intercom API client.
+
+        Args:
+            credentials: Intercom API credentials
+        """
+        self.credentials = credentials
+        self.base_url = credentials.base_url
+
+        # Set up authentication headers
+        self.headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Intercom-Version": API_VERSION,  # REQUIRED header
+        }
+
+        if isinstance(credentials, IntercomCredentialsAccessToken):
+            self.headers["Authorization"] = f"Bearer {credentials.access_token}"
+        elif isinstance(credentials, IntercomCredentialsOAuth):
+            self.headers["Authorization"] = f"Bearer {credentials.oauth_token}"
+        else:
+            raise TypeError(
+                "Invalid credentials type. Must be IntercomCredentialsAccessToken or IntercomCredentialsOAuth"
+            )
+
+        # Create HTTP client with rate limit retry for 429 status codes
+        self.client = create_client(retry_status_codes=[429, 502, 503])
+
+    def _make_request(
+        self,
+        method: str,
+        endpoint: str,
+        params: Optional[Dict[str, Any]] = None,
+        json_data: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Make a request to the Intercom API.
+
+        Args:
+            method: HTTP method (GET, POST, etc.)
+            endpoint: API endpoint path
+            params: Query parameters
+            json_data: JSON body data
+
+        Returns:
+            Response JSON data
+        """
+        url = f"{self.base_url}{endpoint}"
+
+        if method.upper() == "GET":
+            response = self.client.get(url, headers=self.headers, params=params)
+        elif method.upper() == "POST":
+            response = self.client.post(
+                url, headers=self.headers, json=json_data, params=params
+            )
+        else:
+            response = self.client.request(
+                method, url, headers=self.headers, json=json_data, params=params
+            )
+
+        # The create_client already handles rate limiting (429) with retries
+        # Just check for other errors
+        if response.status_code >= 400:
+            error_msg = f"Intercom API error {response.status_code}: {response.text}"
+            raise Exception(error_msg)
+
+        return response.json()
+
+    def get_pages(
+        self,
+        endpoint: str,
+        data_key: str,
+        pagination_type: PaginationType,
+        params: Optional[Dict[str, Any]] = None,
+        search_query: Optional[Dict[str, Any]] = None,
+    ) -> Iterator[TDataItems]:
+        """
+        Get paginated data from an Intercom endpoint.
+
+        Args:
+            endpoint: API endpoint path
+            data_key: Key in response containing the data items
+            pagination_type: Type of pagination to use
+            params: Query parameters
+            search_query: Search query for search endpoints
+
+        Yields:
+            Lists of data items from each page
+        """
+        params = params or {}
+
+        if pagination_type == PaginationType.SIMPLE:
+            # Single page, no pagination
+            response = self._make_request("GET", endpoint, params)
+            if data_key in response:
+                yield response[data_key]
+            return
+
+        elif pagination_type == PaginationType.CURSOR:
+            # Cursor-based pagination
+            params["per_page"] = params.get("per_page", DEFAULT_PAGE_SIZE)
+            next_cursor = None
+
+            while True:
+                if next_cursor:
+                    params["starting_after"] = next_cursor
+
+                response = self._make_request("GET", endpoint, params)
+
+                # Yield the data
+                if data_key in response and response[data_key]:
+                    yield response[data_key]
+
+                # Check for next page
+                pages_info = response.get("pages", {})
+                if not pages_info.get("next"):
+                    break
+
+                next_cursor = pages_info.get("next", {}).get("starting_after")
+                if not next_cursor:
+                    break
+
+        elif pagination_type == PaginationType.SCROLL:
+            # Scroll API pagination (for large exports)
+            scroll_param = None
+
+            while True:
+                scroll_endpoint = endpoint
+                if scroll_param:
+                    scroll_endpoint = f"{endpoint}/scroll"
+                    params = {"scroll_param": scroll_param}
+
+                response = self._make_request("GET", scroll_endpoint, params)
+
+                # Yield the data
+                if data_key in response and response[data_key]:
+                    yield response[data_key]
+
+                # Get next scroll parameter
+                scroll_param = response.get("scroll_param")
+                if not scroll_param:
+                    break
+
+        elif pagination_type == PaginationType.SEARCH:
+            # Search API pagination
+            if not search_query:
+                raise ValueError("Search query required for search pagination")
+
+            pagination_info = search_query.get("pagination", {})
+            pagination_info["per_page"] = pagination_info.get(
+                "per_page", DEFAULT_PAGE_SIZE
+            )
+
+            while True:
+                # Build search request
+                request_data = {
+                    "query": search_query.get("query", {}),
+                    "pagination": pagination_info,
+                }
+
+                if "sort" in search_query:
+                    request_data["sort"] = search_query["sort"]
+
+                response = self._make_request("POST", endpoint, json_data=request_data)
+
+                # Yield the data
+                if data_key in response and response[data_key]:
+                    yield response[data_key]
+
+                # Check for next page
+                pages_info = response.get("pages", {})
+                if not pages_info.get("next"):
+                    break
+
+                next_cursor = pages_info.get("next", {}).get("starting_after")
+                if not next_cursor:
+                    break
+
+                pagination_info["starting_after"] = next_cursor
+
+    def get_single_resource(self, endpoint: str, resource_id: str) -> TDataItem:
+        """
+        Get a single resource by ID.
+
+        Args:
+            endpoint: Base endpoint path
+            resource_id: Resource ID
+
+        Returns:
+            Resource data
+        """
+        return self._make_request("GET", f"{endpoint}/{resource_id}")
+
+    def search(
+        self,
+        resource_type: str,
+        query: Dict[str, Any],
+        sort: Optional[Dict[str, str]] = None,
+    ) -> Iterator[TDataItems]:
+        """
+        Search for resources using the Search API.
+
+        Args:
+            resource_type: Type of resource to search (contacts, companies, conversations)
+            query: Search query following Intercom's query format
+            sort: Optional sort configuration
+
+        Yields:
+            Lists of matching resources
+        """
+        endpoint = f"/{resource_type}/search"
+        search_query = {"query": query}
+
+        if sort:
+            search_query["sort"] = sort
+
+        yield from self.get_pages(
+            endpoint=endpoint,
+            data_key="data",
+            pagination_type=PaginationType.SEARCH,
+            search_query=search_query,
+        )
+
+
+def transform_contact(contact: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Transform a contact record to ensure consistent format.
+
+    Args:
+        contact: Raw contact data from API
+
+    Returns:
+        Transformed contact data
+    """
+    # Ensure consistent field names and types
+    transformed = contact.copy()
+
+    # Flatten location data if present
+    if "location" in transformed and isinstance(transformed["location"], dict):
+        location = transformed.pop("location")
+        transformed["location_country"] = location.get("country")
+        transformed["location_region"] = location.get("region")
+        transformed["location_city"] = location.get("city")
+
+    # Flatten companies relationship
+    if "companies" in transformed and isinstance(transformed["companies"], dict):
+        companies_data = transformed["companies"].get("data", [])
+        transformed["company_ids"] = [
+            c.get("id") for c in companies_data if c.get("id")
+        ]
+        transformed["companies_count"] = len(companies_data)
+
+    # Ensure custom_attributes is always a dict
+    if "custom_attributes" not in transformed:
+        transformed["custom_attributes"] = {}
+
+    return transformed
+
+
+def transform_company(company: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Transform a company record to ensure consistent format.
+
+    Args:
+        company: Raw company data from API
+
+    Returns:
+        Transformed company data
+    """
+    transformed = company.copy()
+
+    # Ensure custom_attributes is always a dict
+    if "custom_attributes" not in transformed:
+        transformed["custom_attributes"] = {}
+
+    # Flatten plan information if it's an object
+    if "plan" in transformed and isinstance(transformed["plan"], dict):
+        plan = transformed.pop("plan")
+        transformed["plan_id"] = plan.get("id")
+        transformed["plan_name"] = plan.get("name")
+
+    return transformed
+
+
+def transform_conversation(conversation: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Transform a conversation record to ensure consistent format.
+
+    Args:
+        conversation: Raw conversation data from API
+
+    Returns:
+        Transformed conversation data
+    """
+    transformed = conversation.copy()
+
+    # Extract statistics if present
+    if "statistics" in transformed and isinstance(transformed["statistics"], dict):
+        stats = transformed.pop("statistics")
+        transformed["first_contact_reply_at"] = stats.get("first_contact_reply_at")
+        transformed["first_admin_reply_at"] = stats.get("first_admin_reply_at")
+        transformed["last_contact_reply_at"] = stats.get("last_contact_reply_at")
+        transformed["last_admin_reply_at"] = stats.get("last_admin_reply_at")
+        transformed["median_admin_reply_time"] = stats.get("median_admin_reply_time")
+        transformed["mean_admin_reply_time"] = stats.get("mean_admin_reply_time")
+
+    # Flatten conversation parts count
+    if "conversation_parts" in transformed and isinstance(
+        transformed["conversation_parts"], dict
+    ):
+        parts = transformed["conversation_parts"]
+        transformed["conversation_parts_count"] = parts.get("total_count", 0)
+
+    return transformed
+
+
+def convert_datetime_to_timestamp(dt_obj: Any) -> int:
+    """
+    Convert datetime object to Unix timestamp for Intercom API compatibility.
+
+    Args:
+        dt_obj: DateTime object (pendulum or datetime)
+
+    Returns:
+        Unix timestamp as integer
+    """
+    if hasattr(dt_obj, "int_timestamp"):
+        return dt_obj.int_timestamp
+    elif hasattr(dt_obj, "timestamp"):
+        return int(dt_obj.timestamp())
+    else:
+        raise ValueError(f"Cannot convert {type(dt_obj)} to timestamp")
+
+
+def create_search_resource(
+    api_client: "IntercomAPIClient",
+    resource_name: str,
+    updated_at_incremental: Any,
+    transform_func: Optional[Callable] = None,
+) -> Iterator[TDataItems]:
+    """
+    Generic function for search-based incremental resources.
+
+    Args:
+        api_client: Intercom API client
+        resource_name: Name of the resource (contacts, conversations)
+        updated_at_incremental: DLT incremental object
+        transform_func: Optional transformation function
+
+    Yields:
+        Transformed resource records
+    """
+    query = build_incremental_query(
+        "updated_at",
+        updated_at_incremental.last_value,
+        updated_at_incremental.end_value,
+    )
+
+    for page in api_client.search(resource_name, query):
+        if transform_func:
+            transformed_items = [transform_func(item) for item in page]
+            yield transformed_items
+        else:
+            yield page
+
+        if updated_at_incremental.end_out_of_range:
+            return
+
+
+def create_tickets_resource(
+    api_client: "IntercomAPIClient",
+    updated_at_incremental: Any,
+) -> Iterator[TDataItems]:
+    """
+    Special function for tickets resource with updated_since parameter.
+
+    Args:
+        api_client: Intercom API client
+        updated_at_incremental: DLT incremental object
+
+    Yields:
+        Filtered ticket records
+    """
+    params = {"updated_since": updated_at_incremental.last_value}
+
+    end_timestamp = (
+        updated_at_incremental.end_value if updated_at_incremental.end_value else None
+    )
+
+    for page in api_client.get_pages(
+        "/tickets", "tickets", PaginationType.CURSOR, params=params
+    ):
+        if end_timestamp:
+            filtered_tickets = [
+                t for t in page if t.get("updated_at", 0) <= end_timestamp
+            ]
+            if filtered_tickets:
+                yield filtered_tickets
+
+            if any(t.get("updated_at", 0) > end_timestamp for t in page):
+                return
+        else:
+            yield page
+
+
+def create_pagination_resource(
+    api_client: "IntercomAPIClient",
+    endpoint: str,
+    data_key: str,
+    pagination_type: PaginationType,
+    updated_at_incremental: Any,
+    transform_func: Optional[Callable] = None,
+    params: Optional[Dict[str, Any]] = None,
+) -> Iterator[TDataItems]:
+    """
+    Generic function for cursor/simple pagination with client-side filtering.
+
+    Args:
+        api_client: Intercom API client
+        endpoint: API endpoint path
+        data_key: Key in response containing data
+        pagination_type: Type of pagination
+        updated_at_incremental: DLT incremental object
+        transform_func: Optional transformation function
+        params: Additional query parameters
+
+    Yields:
+        Filtered and transformed resource records
+    """
+    for page in api_client.get_pages(
+        endpoint, data_key, pagination_type, params=params
+    ):
+        filtered_items = []
+        for item in page:
+            item_updated = item.get("updated_at", 0)
+            if item_updated >= updated_at_incremental.last_value:
+                if (
+                    updated_at_incremental.end_value
+                    and item_updated > updated_at_incremental.end_value
+                ):
+                    continue
+
+                if transform_func:
+                    filtered_items.append(transform_func(item))
+                else:
+                    filtered_items.append(item)
+
+        if filtered_items:
+            yield filtered_items
+
+        if updated_at_incremental.end_out_of_range:
+            return
+
+
+def create_resource_from_config(
+    resource_name: str,
+    config: Dict[str, Any],
+    api_client: "IntercomAPIClient",
+    start_timestamp: int,
+    end_timestamp: Optional[int],
+    transform_functions: Dict[str, Callable],
+) -> Any:
+    """
+    Create a DLT resource from configuration.
+
+    Args:
+        resource_name: Name of the resource
+        config: Resource configuration dict
+        api_client: Intercom API client
+        start_timestamp: Start timestamp for incremental loading
+        end_timestamp: End timestamp for incremental loading
+        transform_functions: Dict mapping transform function names to actual functions
+
+    Returns:
+        DLT resource function
+    """
+    import dlt
+
+    # Determine write disposition
+    write_disposition = "merge" if config["incremental"] else "replace"
+
+    # Get transform function if specified
+    transform_func = None
+    if config.get("transform_func"):
+        transform_func = transform_functions.get(config["transform_func"])
+
+    def resource_function(
+        updated_at: Optional[dlt.sources.incremental[int]] = dlt.sources.incremental(
+            "updated_at",
+            initial_value=start_timestamp,
+            end_value=end_timestamp,
+            allow_external_schedulers=True,
+        )
+        if config["incremental"]
+        else None,
+    ) -> Iterator[TDataItems]:
+        """
+        Auto-generated resource function.
+        """
+        resource_type = config["type"]
+
+        if resource_type == "search":
+            yield from create_search_resource(
+                api_client, resource_name, updated_at, transform_func
+            )
+        elif resource_type == "pagination":
+            yield from create_pagination_resource(
+                api_client,
+                config["endpoint"],
+                config["data_key"],
+                getattr(PaginationType, config["pagination_type"].upper()),
+                updated_at,
+                transform_func,
+                config.get("params"),
+            )
+        elif resource_type == "tickets":
+            yield from create_tickets_resource(api_client, updated_at)
+        elif resource_type == "simple":
+            # Non-incremental resources
+            yield from api_client.get_pages(
+                config["endpoint"],
+                config["data_key"],
+                getattr(PaginationType, config["pagination_type"].upper()),
+            )
+        else:
+            raise ValueError(f"Unknown resource type: {resource_type}")
+
+    # For non-incremental resources, we need to return a function without parameters
+    if not config["incremental"]:
+
+        @dlt.resource(
+            name=resource_name,
+            primary_key="id",
+            write_disposition="replace",
+            columns=config.get("columns", {}),
+        )
+        def simple_resource_function() -> Iterator[TDataItems]:
+            """
+            Auto-generated simple resource function.
+            """
+            yield from api_client.get_pages(
+                config["endpoint"],
+                config["data_key"],
+                getattr(PaginationType, config["pagination_type"].upper()),
+            )
+
+        return simple_resource_function
+
+    # Apply the decorator to the function
+    return dlt.resource(  # type: ignore[call-overload]
+        resource_function,
+        name=resource_name,
+        primary_key="id",
+        write_disposition=write_disposition,
+        columns=config.get("columns", {}),
+    )
+
+
+def build_incremental_query(
+    field: str,
+    start_value: Any,
+    end_value: Optional[Any] = None,
+) -> Dict[str, Any]:
+    """
+    Build a search query for incremental loading.
+
+    Args:
+        field: Field to filter on
+        start_value: Start value (inclusive)
+        end_value: Optional end value (inclusive)
+
+    Returns:
+        Query dict for Intercom Search API
+    """
+    conditions = [
+        {
+            "field": field,
+            "operator": ">",
+            "value": start_value,
+        }
+    ]
+
+    if end_value is not None:
+        conditions.append(
+            {
+                "field": field,
+                "operator": "<",
+                "value": end_value,
+            }
+        )
+
+    if len(conditions) == 1:
+        return conditions[0]
+    else:
+        return {
+            "operator": "AND",
+            "value": conditions,
+        }

ingestr/src/intercom/settings.py

@@ -0,0 +1,277 @@
+"""
+Configuration settings and constants for Intercom API integration.
+"""
+
+from datetime import datetime
+from typing import Dict, List, Tuple
+
+# API Version - REQUIRED for all requests
+API_VERSION = "2.14"
+
+# Default start date for incremental loading
+DEFAULT_START_DATE = datetime(2020, 1, 1)
+
+# Pagination settings
+DEFAULT_PAGE_SIZE = 150
+MAX_PAGE_SIZE = 150  # Intercom's maximum
+SCROLL_EXPIRY_SECONDS = 60  # Scroll sessions expire after 1 minute
+
+# Rate limiting settings
+RATE_LIMIT_PER_10_SECONDS = 166
+RATE_LIMIT_RETRY_AFTER_DEFAULT = 10
+
+# Regional API endpoints
+REGIONAL_ENDPOINTS = {
+    "us": "https://api.intercom.io",
+    "eu": "https://api.eu.intercom.io",
+    "au": "https://api.au.intercom.io",
+}
+
+# Resource configuration for automatic generation
+# Format: resource_name -> config dict
+RESOURCE_CONFIGS = {
+    # Search-based incremental resources
+    "contacts": {
+        "type": "search",
+        "incremental": True,
+        "transform_func": "transform_contact",
+        "columns": {
+            "custom_attributes": {"data_type": "json"},
+            "tags": {"data_type": "json"},
+        },
+    },
+    "conversations": {
+        "type": "search",
+        "incremental": True,
+        "transform_func": "transform_conversation",
+        "columns": {
+            "custom_attributes": {"data_type": "json"},
+            "tags": {"data_type": "json"},
+        },
+    },
+    # Pagination-based incremental resources
+    "companies": {
+        "type": "pagination",
+        "endpoint": "/companies",
+        "data_key": "data",
+        "pagination_type": "cursor",
+        "incremental": True,
+        "transform_func": "transform_company",
+        "params": {"per_page": 50},
+        "columns": {
+            "custom_attributes": {"data_type": "json"},
+            "tags": {"data_type": "json"},
+        },
+    },
+    "articles": {
+        "type": "pagination",
+        "endpoint": "/articles",
+        "data_key": "data",
+        "pagination_type": "cursor",
+        "incremental": True,
+        "transform_func": None,
+        "params": None,
+        "columns": {},
+    },
+    # Special case - tickets
+    "tickets": {
+        "type": "tickets",
+        "incremental": True,
+        "transform_func": None,
+        "columns": {
+            "ticket_attributes": {"data_type": "json"},
+        },
+    },
+    # Simple replace resources (non-incremental)
+    "tags": {
+        "type": "simple",
+        "endpoint": "/tags",
+        "data_key": "data",
+        "pagination_type": "simple",
+        "incremental": False,
+        "transform_func": None,
+        "columns": {},
+    },
+    "segments": {
+        "type": "simple",
+        "endpoint": "/segments",
+        "data_key": "segments",
+        "pagination_type": "cursor",
+        "incremental": False,
+        "transform_func": None,
+        "columns": {},
+    },
+    "teams": {
+        "type": "simple",
+        "endpoint": "/teams",
+        "data_key": "teams",
+        "pagination_type": "simple",
+        "incremental": False,
+        "transform_func": None,
+        "columns": {},
+    },
+    "admins": {
+        "type": "simple",
+        "endpoint": "/admins",
+        "data_key": "admins",
+        "pagination_type": "simple",
+        "incremental": False,
+        "transform_func": None,
+        "columns": {},
+    },
+    "data_attributes": {
+        "type": "simple",
+        "endpoint": "/data_attributes",
+        "data_key": "data",
+        "pagination_type": "cursor",
+        "incremental": False,
+        "transform_func": None,
+        "columns": {},
+    },
+}
+
+# Core endpoints with their configuration (kept for backwards compatibility)
+# Format: (endpoint_path, data_key, supports_incremental, pagination_type)
+CORE_ENDPOINTS: Dict[str, Tuple[str, str, bool, str]] = {
+    "contacts": ("/contacts", "data", True, "cursor"),
+    "companies": ("/companies", "data", True, "cursor"),
+    "conversations": ("/conversations", "conversations", True, "cursor"),
+    "tickets": ("/tickets", "tickets", True, "cursor"),
+    "admins": ("/admins", "admins", False, "simple"),
+    "teams": ("/teams", "teams", False, "simple"),
+    "tags": ("/tags", "data", False, "simple"),
+    "segments": ("/segments", "segments", False, "cursor"),
+    "articles": ("/articles", "data", True, "cursor"),
+    "collections": ("/help_center/collections", "data", False, "cursor"),
+    "data_attributes": ("/data_attributes", "data", False, "cursor"),
+}
+
+# Incremental endpoints using search API
+SEARCH_ENDPOINTS: Dict[str, str] = {
+    "contacts_search": "/contacts/search",
+    "companies_search": "/companies/search",
+    "conversations_search": "/conversations/search",
+}
+
+# Special endpoints requiring different handling
+SCROLL_ENDPOINTS: List[str] = [
+    "companies",  # Can use scroll for large exports
+]
+
+# Event tracking endpoint
+EVENTS_ENDPOINT = "/events"
+
+# Ticket fields endpoint for custom field mapping
+TICKET_FIELDS_ENDPOINT = "/ticket_types/{ticket_type_id}/attributes"
+
+# Default fields to retrieve for each resource type
+DEFAULT_CONTACT_FIELDS = [
+    "id",
+    "type",
+    "external_id",
+    "email",
+    "phone",
+    "name",
+    "created_at",
+    "updated_at",
+    "signed_up_at",
+    "last_seen_at",
+    "last_contacted_at",
+    "last_email_opened_at",
+    "last_email_clicked_at",
+    "browser",
+    "browser_language",
+    "browser_version",
+    "location",
+    "os",
+    "role",
+    "custom_attributes",
+    "tags",
+    "companies",
+]
+
+DEFAULT_COMPANY_FIELDS = [
+    "id",
+    "type",
+    "company_id",
+    "name",
+    "plan",
+    "size",
+    "website",
+    "industry",
+    "created_at",
+    "updated_at",
+    "monthly_spend",
+    "session_count",
+    "user_count",
+    "custom_attributes",
+    "tags",
+]
+
+DEFAULT_CONVERSATION_FIELDS = [
+    "id",
+    "type",
+    "created_at",
+    "updated_at",
+    "waiting_since",
+    "snoozed_until",
+    "state",
+    "open",
+    "read",
+    "priority",
+    "admin_assignee_id",
+    "team_assignee_id",
+    "tags",
+    "conversation_rating",
+    "source",
+    "contacts",
+    "teammates",
+    "custom_attributes",
+    "first_contact_reply",
+    "sla_applied",
+    "statistics",
+    "conversation_parts",
+]
+
+DEFAULT_TICKET_FIELDS = [
+    "id",
+    "type",
+    "ticket_id",
+    "category",
+    "ticket_attributes",
+    "ticket_state",
+    "ticket_type",
+    "created_at",
+    "updated_at",
+    "ticket_parts",
+    "contacts",
+    "admin_assignee_id",
+    "team_assignee_id",
+    "open",
+    "snoozed_until",
+]
+
+# Resources that support custom attributes
+SUPPORTS_CUSTOM_ATTRIBUTES = [
+    "contacts",
+    "companies",
+    "conversations",
+]
+
+# Maximum limits
+MAX_CUSTOM_ATTRIBUTES_PER_RESOURCE = 100
+MAX_EVENT_TYPES_PER_WORKSPACE = 120
+MAX_CONVERSATION_PARTS = 500
+MAX_SEARCH_RESULTS = 10000
+
+# Field type mapping for custom attributes
+INTERCOM_TO_DLT_TYPE_MAPPING = {
+    "string": "text",
+    "integer": "bigint",
+    "float": "double",
+    "boolean": "bool",
+    "date": "timestamp",
+    "datetime": "timestamp",
+    "object": "json",
+    "list": "json",
+}

ingestr/src/sources.py

@@ -3703,3 +3703,76 @@ class AnthropicSource:
             initial_start_date=start_date,
             end_date=end_date,
         ).with_resources(table)
+
+
+class IntercomSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        # intercom://?access_token=<token>&region=<us|eu|au>
+        # OR intercom://?oauth_token=<token>&region=<us|eu|au>
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+
+        # Check for authentication
+        access_token = params.get("access_token")
+        oauth_token = params.get("oauth_token")
+        region = params.get("region", ["us"])[0]
+
+        if not access_token and not oauth_token:
+            raise MissingValueError("access_token or oauth_token", "Intercom")
+
+        # Validate table/resource
+        supported_tables = [
+            "contacts",
+            "companies",
+            "conversations",
+            "tickets",
+            "tags",
+            "segments",
+            "teams",
+            "admins",
+            "articles",
+            "data_attributes",
+        ]
+
+        if table not in supported_tables:
+            raise UnsupportedResourceError(table, "Intercom")
+
+        # Get date parameters
+        start_date = kwargs.get("interval_start")
+        if start_date:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.datetime(2020, 1, 1)
+
+        end_date = kwargs.get("interval_end")
+        if end_date:
+            end_date = ensure_pendulum_datetime(end_date)
+
+        # Import and initialize the source
+        from ingestr.src.intercom import (
+            IntercomCredentialsAccessToken,
+            IntercomCredentialsOAuth,
+            TIntercomCredentials,
+            intercom_source,
+        )
+
+        credentials: TIntercomCredentials
+        if access_token:
+            credentials = IntercomCredentialsAccessToken(
+                access_token=access_token[0], region=region
+            )
+        else:
+            if not oauth_token:
+                raise MissingValueError("oauth_token", "Intercom")
+            credentials = IntercomCredentialsOAuth(
+                oauth_token=oauth_token[0], region=region
+            )
+
+        return intercom_source(
+            credentials=credentials,
+            start_date=start_date,
+            end_date=end_date,
+        ).with_resources(table)

{ingestr-0.14.0.dist-info → ingestr-0.14.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.0
+Version: 0.14.2
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.0.dist-info → ingestr-0.14.2.dist-info}/RECORD

@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=wnJ_CPlYtFn5F7P7HRnnyrCPI2u2P6bB5qMLZpfYxkc,20
+ingestr/src/buildinfo.py,sha256=etAF7mJbjXgR22CNwj2pF9fzWwF_eMqGhIYdwvHnftw,20
 ingestr/src/destinations.py,sha256=BE8tK69Aq7FLBV_j6ijMhZCRIUYpSTnk7FTZMZW5Q5M,28557
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=sT5x4R95Qa0QIjkS7a5lFKUFwFpVpdb1nv8HIxOu_RM,7376
+ingestr/src/factory.py,sha256=u6IxjM9Rt6klr0Sa44OiDO-zFTJMvp84hFX3vZKAP4M,7432
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=ADnE41x6-9U0fK1CYqPc6XRfGqg_6mZGqP10tlLUxDU,130361
+ingestr/src/sources.py,sha256=gcweUJVZ4ScQzSeu_pCm4kgsLUymNuF6K0qWy6OE6OM,132655
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -56,8 +56,8 @@ ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5v
 ingestr/src/filesystem/__init__.py,sha256=42YAOHQxZ7TkTXC1eeaLUJpjqJ3l7DH7C8j927pV4pc,4353
 ingestr/src/filesystem/helpers.py,sha256=bg0muSHZr3hMa8H4jN2-LGWzI-SUoKlQNiWJ74-YYms,3211
 ingestr/src/filesystem/readers.py,sha256=a0fKkaRpnAOGsXI3EBNYZa7x6tlmAOsgRzb883StY30,3987
-ingestr/src/fluxx/__init__.py,sha256=Ua_QVKfeNWzpJRaGgjgR50Ot52s0f5hY-VN-a53gY2Y,566245
-ingestr/src/fluxx/helpers.py,sha256=dCNgvMMTSEO4LNp6luNZ-XrV4NPW-_OUfmp0k3jFhuc,6602
+ingestr/src/fluxx/__init__.py,sha256=xDtkqt4_f9LgVNt-APCQ7tsJ8gM6msEroyKAUToNC8A,566053
+ingestr/src/fluxx/helpers.py,sha256=zJmlQWwiv9snnLqTygiWVZy7-0rGi_K427hRUuZeHEM,6352
 ingestr/src/frankfurter/__init__.py,sha256=gOdL8ZqgHHYZByjtfE3WX3BTRHdYqyn9FpQwzDHSAx0,5089
 ingestr/src/frankfurter/helpers.py,sha256=SpRr992OcSf7IDI5y-ToUdO6m6sGpqFz59LTY0ojchI,1502
 ingestr/src/freshdesk/__init__.py,sha256=ukyorgCNsW_snzsYBDsr3Q0WB8f-to9Fk0enqHHFQlk,3087
@@ -88,6 +88,9 @@ ingestr/src/hubspot/helpers.py,sha256=k2b-lhxqBNKHoOSHoHegFSsk8xxjjGA0I04V0XyX2b
 ingestr/src/hubspot/settings.py,sha256=i73MkSiJfRLMFLfiJgYdhp-rhymHTfoqFzZ4uOJdFJM,2456
 ingestr/src/influxdb/__init__.py,sha256=cYsGnDPNHRTe9pp14ogDQgPTCI9TOdyJm1MaNuQLHdk,1290
 ingestr/src/influxdb/client.py,sha256=hCxSNREAWWEvvAV3RQbKaWp2-e_7EE8xmVRjTwLFEFo,1230
+ingestr/src/intercom/__init__.py,sha256=rqorWFwcfcTYrCrpSsPPM2sGOc7qq5XbYZRCDVJXjyI,4451
+ingestr/src/intercom/helpers.py,sha256=IljM0x4K70nuahidZaP7mtIlsHkPIcZq56j9mmuSck4,21074
+ingestr/src/intercom/settings.py,sha256=BU-jmlIOA2BrR-mWyjL6QXhiNqdY3WrNjzaWd5MCKLk,6983
 ingestr/src/isoc_pulse/__init__.py,sha256=9b4eN4faatpiwTuRNPuYcEt1hEFDEjua9XhfakUigBk,4648
 ingestr/src/kafka/__init__.py,sha256=QUHsGmdv5_E-3z0GDHXvbk39puwuGDBsyYSDhvbA89E,3595
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
@@ -166,8 +169,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.0.dist-info/METADATA,sha256=rRaZKV5Ik4cZgl0s6BPRvnrSd5fs7FN0ptg1hNCjKT0,15265
-ingestr-0.14.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.0.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.0.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.0.dist-info/RECORD,,
+ingestr-0.14.2.dist-info/METADATA,sha256=-UyaGpKgZSYETEBSfKLzRXn5vG3VA9qmofpmoD-gGa0,15265
+ingestr-0.14.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.2.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.2.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.2.dist-info/RECORD,,