ingestr 0.13.2__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/linear/helpers.py

@@ -0,0 +1,111 @@
+from typing import Any, Dict, Iterator, Optional
+
+import dlt
+import pendulum
+import requests
+
+LINEAR_GRAPHQL_ENDPOINT = "https://api.linear.app/graphql"
+
+
+def _graphql(
+    api_key: str, query: str, variables: Optional[Dict[str, Any]] = None
+) -> Dict[str, Any]:
+    headers = {"Authorization": api_key, "Content-Type": "application/json"}
+    response = requests.post(
+        LINEAR_GRAPHQL_ENDPOINT,
+        json={"query": query, "variables": variables or {}},
+        headers=headers,
+    )
+    response.raise_for_status()
+    payload = response.json()
+    if "errors" in payload:
+        raise ValueError(str(payload["errors"]))
+    return payload["data"]
+
+
+def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
+    cursor: Optional[str] = None
+    while True:
+        data = _graphql(api_key, query, {"cursor": cursor})[root]
+        for item in data["nodes"]:
+            yield item
+        if not data["pageInfo"]["hasNextPage"]:
+            break
+        cursor = data["pageInfo"]["endCursor"]
+
+
+def _get_date_range(updated_at, start_date):
+    """Extract current start and end dates from incremental state."""
+    if updated_at.last_value:
+        current_start_date = pendulum.parse(updated_at.last_value)
+    else:
+        current_start_date = pendulum.parse(start_date)
+
+    if updated_at.end_value:
+        current_end_date = pendulum.parse(updated_at.end_value)
+    else:
+        current_end_date = pendulum.now(tz="UTC")
+
+    return current_start_date, current_end_date
+
+
+def _paginated_resource(
+    api_key: str, query: str, query_field: str, updated_at, start_date
+) -> Iterator[Dict[str, Any]]:
+    """Helper function for paginated resources with date filtering."""
+    current_start_date, current_end_date = _get_date_range(updated_at, start_date)
+
+    for item in _paginate(api_key, query, query_field):
+        if pendulum.parse(item["updatedAt"]) >= current_start_date:
+            if pendulum.parse(item["updatedAt"]) <= current_end_date:
+                yield normalize_dictionaries(item)
+
+
+def _create_paginated_resource(
+    resource_name: str,
+    query: str,
+    query_field: str,
+    api_key: str,
+    start_date,
+    end_date=None,
+):
+    """Factory function to create paginated resources dynamically."""
+
+    @dlt.resource(name=resource_name, primary_key="id", write_disposition="merge")
+    def paginated_resource(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            "updatedAt",
+            initial_value=start_date.isoformat(),
+            end_value=end_date.isoformat() if end_date else None,
+            range_start="closed",
+            range_end="closed",
+        ),
+    ) -> Iterator[Dict[str, Any]]:
+        for item in _paginated_resource(
+            api_key, query, query_field, updated_at, start_date
+        ):
+            yield normalize_dictionaries(item)
+
+    return paginated_resource
+
+
+def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Automatically normalize dictionary fields by detecting their structure:
+    - Convert nested objects with 'id' field to {field_name}_id
+    - Convert objects with 'nodes' field to arrays
+
+    """
+    normalized_item = item.copy()
+
+    for key, value in list(normalized_item.items()):
+        if isinstance(value, dict):
+            # If the dict has an 'id' field, replace with {key}_id
+            if "id" in value:
+                normalized_item[f"{key}_id"] = value["id"]
+                del normalized_item[key]
+            # If the dict has 'nodes' field, extract the nodes array
+            elif "nodes" in value:
+                normalized_item[key] = value["nodes"]
+
+    return normalized_item

ingestr/src/linkedin_ads/helpers.py

@@ -18,7 +18,6 @@ def retry_on_limit(
 
 def create_client() -> requests.Session:
     return Client(
-        request_timeout=10.0,
         raise_for_status=False,
         retry_condition=retry_on_limit,
         request_max_attempts=12,

ingestr/src/loader.py

@@ -0,0 +1,69 @@
+import csv
+import gzip
+import json
+import subprocess
+from contextlib import contextmanager
+from typing import Generator
+
+from pyarrow.parquet import ParquetFile  # type: ignore
+
+PARQUET_BATCH_SIZE = 64
+
+
+class UnsupportedLoaderFileFormat(Exception):
+    pass
+
+
+def load_dlt_file(filepath: str) -> Generator:
+    """
+    load_dlt_file reads dlt loader files. It handles different loader file formats
+    automatically. It returns a generator that yield data items as a python dict
+    """
+    result = subprocess.run(
+        ["file", "-b", filepath],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    filetype = result.stdout.strip()
+    with factory(filetype, filepath) as reader:
+        yield from reader
+
+
+def factory(filetype: str, filepath: str):
+    # ???(turtledev): can dlt produce non-gizpped jsonl files?
+    if filetype.startswith("gzip"):
+        return jsonlfile(filepath)
+    elif filetype.startswith("CSV"):
+        return csvfile(filepath)
+    elif filetype.startswith("Apache Parquet"):
+        return parquetfile(filepath)
+    else:
+        raise UnsupportedLoaderFileFormat(filetype)
+
+
+@contextmanager
+def jsonlfile(filepath: str):
+    def reader(fd):
+        for line in fd:
+            yield json.loads(line.decode().strip())
+
+    with gzip.open(filepath) as fd:
+        yield reader(fd)
+
+
+@contextmanager
+def csvfile(filepath: str):
+    with open(filepath, "r") as fd:
+        yield csv.DictReader(fd)
+
+
+@contextmanager
+def parquetfile(filepath: str):
+    def reader(pf: ParquetFile):
+        for batch in pf.iter_batches(PARQUET_BATCH_SIZE):
+            yield from batch.to_pylist()
+
+    with open(filepath, "rb") as fd:
+        yield reader(ParquetFile(fd))

ingestr/src/mailchimp/__init__.py

@@ -0,0 +1,126 @@
+"""
+Mailchimp source for data extraction via REST API.
+
+This source provides access to Mailchimp account data.
+"""
+
+from typing import Any, Iterable, Iterator
+
+import dlt
+from dlt.sources import DltResource
+
+from ingestr.src.http_client import create_client
+from ingestr.src.mailchimp.helpers import (
+    create_merge_resource,
+    create_nested_resource,
+    create_replace_resource,
+)
+from ingestr.src.mailchimp.settings import (
+    MERGE_ENDPOINTS,
+    NESTED_ENDPOINTS,
+    REPLACE_ENDPOINTS,
+)
+
+
+@dlt.source(max_table_nesting=0, name="mailchimp_source")
+def mailchimp_source(
+    api_key: str,
+    server: str,
+) -> Iterable[DltResource]:
+    """
+    Mailchimp data source.
+
+    Args:
+        api_key: Mailchimp API key for authentication
+        server: Server prefix (e.g., 'us10')
+
+    Yields:
+        DltResource: Data resources for Mailchimp data
+    """
+    base_url = f"https://{server}.api.mailchimp.com/3.0"
+    session = create_client()
+    auth = ("anystring", api_key)
+
+    @dlt.resource(
+        name="account",
+        write_disposition="replace",
+    )
+    def fetch_account() -> Iterator[dict[str, Any]]:
+        """
+        Fetch account information from Mailchimp.
+
+        Table format: account (no parameters needed)
+        """
+        response = session.get(f"{base_url}/", auth=auth)
+        response.raise_for_status()
+        data = response.json()
+        yield data
+
+    # Create resources dynamically
+    resources = [fetch_account]
+
+    # Create merge resources (with incremental loading)
+    for (
+        resource_name,
+        endpoint_path,
+        data_key,
+        primary_key,
+        incremental_key,
+    ) in MERGE_ENDPOINTS:
+        resources.append(
+            create_merge_resource(
+                base_url,
+                session,
+                auth,
+                resource_name,
+                endpoint_path,
+                data_key,
+                primary_key,
+                incremental_key,
+            )
+        )
+
+    # Create replace resources (without incremental loading)
+    for replace_endpoint in REPLACE_ENDPOINTS:
+        resource_name, endpoint_path, data_key, pk = replace_endpoint
+        resources.append(
+            create_replace_resource(
+                base_url,
+                session,
+                auth,
+                resource_name,
+                endpoint_path,
+                data_key,
+                pk,
+            )
+        )
+
+    # Create nested resources (depend on parent resources)
+    for nested_endpoint in NESTED_ENDPOINTS:
+        (
+            parent_name,
+            parent_path,
+            parent_key,
+            parent_id_field,
+            nested_name,
+            nested_path,
+            nested_key,
+            pk,
+        ) = nested_endpoint
+        resources.append(
+            create_nested_resource(
+                base_url,
+                session,
+                auth,
+                parent_name,
+                parent_path,
+                parent_key,
+                parent_id_field,
+                nested_name,
+                nested_path,
+                nested_key,
+                pk,
+            )
+        )
+
+    return tuple(resources)

ingestr/src/mailchimp/helpers.py

@@ -0,0 +1,226 @@
+"""
+Helper functions for Mailchimp source.
+"""
+
+from typing import Any, Iterator
+
+import dlt
+
+
+def fetch_paginated(
+    session,
+    url: str,
+    auth: tuple,
+    data_key: str | None = None,
+) -> Iterator[dict[str, Any]]:
+    """
+    Helper function to fetch paginated data from Mailchimp API.
+
+    Args:
+        session: HTTP session
+        url: API endpoint URL
+        auth: Authentication tuple
+        data_key: Key in response containing the data array (if None, return whole response)
+
+    Yields:
+        Individual items from the paginated response
+    """
+    offset = 0
+    count = 1000  # Maximum allowed by Mailchimp
+
+    while True:
+        params = {"count": count, "offset": offset}
+        response = session.get(url, auth=auth, params=params)
+        response.raise_for_status()
+        data = response.json()
+
+        # Extract items from response
+        if data_key and data_key in data:
+            items = data[data_key]
+        elif isinstance(data, list):
+            items = data
+        else:
+            # If no data_key specified and response is dict, yield the whole response
+            yield data
+            break
+
+        if not items:
+            break
+
+        yield from items
+
+        # Check if we've received fewer items than requested (last page)
+        if len(items) < count:
+            break
+
+        offset += count
+
+
+def create_merge_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    name: str,
+    path: str,
+    key: str,
+    pk: str,
+    ik: str,
+):
+    """
+    Create a DLT resource with merge disposition for incremental loading.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        name: Resource name
+        path: API endpoint path
+        key: Data key in response
+        pk: Primary key field
+        ik: Incremental key field
+
+    Returns:
+        DLT resource function
+    """
+
+    @dlt.resource(
+        name=name,
+        write_disposition="merge",
+        primary_key=pk,
+    )
+    def fetch_data(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            ik, initial_value=None
+        ),
+    ) -> Iterator[dict[str, Any]]:
+        url = f"{base_url}/{path}"
+        yield from fetch_paginated(session, url, auth, data_key=key)
+
+    return fetch_data
+
+
+def create_replace_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    name: str,
+    path: str,
+    key: str,
+    pk: str | None,
+):
+    """
+    Create a DLT resource with replace disposition.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        name: Resource name
+        path: API endpoint path
+        key: Data key in response
+        pk: Primary key field (optional)
+
+    Returns:
+        DLT resource function
+    """
+
+    def fetch_data() -> Iterator[dict[str, Any]]:
+        url = f"{base_url}/{path}"
+        yield from fetch_paginated(session, url, auth, data_key=key)
+
+    # Apply the resource decorator with conditional primary_key
+    if pk is not None:
+        return dlt.resource(
+            fetch_data,
+            name=name,
+            write_disposition="replace",
+            primary_key=pk,
+        )
+    else:
+        return dlt.resource(
+            fetch_data,
+            name=name,
+            write_disposition="replace",
+        )
+
+
+def create_nested_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    parent_resource_name: str,
+    parent_path: str,
+    parent_key: str,
+    parent_id_field: str,
+    nested_name: str,
+    nested_path: str,
+    nested_key: str | None,
+    pk: str | None,
+):
+    """
+    Create a nested DLT resource that depends on a parent resource.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        parent_resource_name: Name of the parent resource
+        parent_path: Parent API endpoint path
+        parent_key: Data key in parent response
+        parent_id_field: Field name for parent ID
+        nested_name: Nested resource name
+        nested_path: Nested API endpoint path (with {id} placeholder)
+        nested_key: Data key in nested response (None to return whole response)
+        pk: Primary key field (optional)
+
+    Returns:
+        DLT resource function
+    """
+
+    def fetch_nested_data() -> Iterator[dict[str, Any]]:
+        # First, fetch parent items
+        parent_url = f"{base_url}/{parent_path}"
+        parent_items = fetch_paginated(session, parent_url, auth, data_key=parent_key)
+
+        # For each parent item, fetch nested data
+        for parent_item in parent_items:
+            parent_id = parent_item.get(parent_id_field)
+            if parent_id:
+                # Build nested URL with parent ID
+                nested_url = f"{base_url}/{nested_path.format(id=parent_id)}"
+
+                # Fetch nested data
+                response = session.get(nested_url, auth=auth)
+                response.raise_for_status()
+                data = response.json()
+
+                # Extract nested items or return whole response
+                if nested_key and nested_key in data:
+                    items = data[nested_key]
+                    if isinstance(items, list):
+                        for item in items:
+                            # Add parent reference
+                            item[f"{parent_resource_name}_id"] = parent_id
+                            yield item
+                    else:
+                        items[f"{parent_resource_name}_id"] = parent_id
+                        yield items
+                else:
+                    # Return whole response with parent reference
+                    data[f"{parent_resource_name}_id"] = parent_id
+                    yield data
+
+    # Apply the resource decorator with conditional primary_key
+    if pk is not None:
+        return dlt.resource(
+            fetch_nested_data,
+            name=nested_name,
+            write_disposition="replace",
+            primary_key=pk,
+        )
+    else:
+        return dlt.resource(
+            fetch_nested_data,
+            name=nested_name,
+            write_disposition="replace",
+        )

ingestr/src/mailchimp/settings.py

@@ -0,0 +1,164 @@
+"""
+Mailchimp API endpoint configurations.
+"""
+
+# Endpoints with merge disposition (have both primary_key and incremental_key)
+# Format: (resource_name, endpoint_path, data_key, primary_key, incremental_key)
+MERGE_ENDPOINTS = [
+    ("audiences", "lists", "lists", "id", "date_created"),
+    ("automations", "automations", "automations", "id", "create_time"),
+    ("campaigns", "campaigns", "campaigns", "id", "create_time"),
+    ("connected_sites", "connected-sites", "sites", "id", "updated_at"),
+    ("conversations", "conversations", "conversations", "id", "last_message.timestamp"),
+    ("ecommerce_stores", "ecommerce/stores", "stores", "id", "updated_at"),
+    ("facebook_ads", "facebook-ads", "facebook_ads", "id", "updated_at"),
+    ("landing_pages", "landing-pages", "landing_pages", "id", "updated_at"),
+    ("reports", "reports", "reports", "id", "send_time"),
+]
+
+# Endpoints with replace disposition
+# Format: (resource_name, endpoint_path, data_key, primary_key)
+REPLACE_ENDPOINTS: list[tuple[str, str, str, str | None]] = [
+    ("account_exports", "account-exports", "exports", None),
+    ("authorized_apps", "authorized-apps", "apps", "id"),
+    ("batches", "batches", "batches", None),
+    ("campaign_folders", "campaign-folders", "folders", "id"),
+    ("chimp_chatter", "activity-feed/chimp-chatter", "chimp_chatter", None),
+]
+
+# Nested endpoints (depend on parent resources)
+# Format: (parent_name, parent_path, parent_key, parent_id_field, nested_name, nested_path, nested_key, pk)
+NESTED_ENDPOINTS: list[tuple[str, str, str, str, str, str, str | None, str | None]] = [
+    # Reports nested endpoints
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_advice",
+        "reports/{id}/advice",
+        None,
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_domain_performance",
+        "reports/{id}/domain-performance",
+        "domains",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_locations",
+        "reports/{id}/locations",
+        "locations",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_sent_to",
+        "reports/{id}/sent-to",
+        "sent_to",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_sub_reports",
+        "reports/{id}/sub-reports",
+        None,
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_unsubscribed",
+        "reports/{id}/unsubscribed",
+        "unsubscribes",
+        None,
+    ),
+    # Lists/Audiences nested endpoints
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_activity",
+        "lists/{id}/activity",
+        "activity",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_clients",
+        "lists/{id}/clients",
+        "clients",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_growth_history",
+        "lists/{id}/growth-history",
+        "history",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_interest_categories",
+        "lists/{id}/interest-categories",
+        "categories",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_locations",
+        "lists/{id}/locations",
+        "locations",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_merge_fields",
+        "lists/{id}/merge-fields",
+        "merge_fields",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_segments",
+        "lists/{id}/segments",
+        "segments",
+        None,
+    ),
+]