ingestr 0.14.9__py3-none-any.whl → 0.14.91__py3-none-any.whl

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.14.9"
+version = "v0.14.91"

ingestr/src/factory.py

@@ -64,6 +64,7 @@ from ingestr.src.sources import (
     LinearSource,
     LinkedInAdsSource,
     LocalCsvSource,
+    MailchimpSource,
     MixpanelSource,
     MondaySource,
     MongoDbSource,
@@ -219,6 +220,7 @@ class SourceDestinationFactory:
         "wise": WiseSource,
         "plusvibeai": PlusVibeAISource,
         "monday": MondaySource,
+        "mailchimp": MailchimpSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/mailchimp/__init__.py

@@ -0,0 +1,126 @@
+"""
+Mailchimp source for data extraction via REST API.
+
+This source provides access to Mailchimp account data.
+"""
+
+from typing import Any, Iterable, Iterator
+
+import dlt
+from dlt.sources import DltResource
+
+from ingestr.src.http_client import create_client
+from ingestr.src.mailchimp.helpers import (
+    create_merge_resource,
+    create_nested_resource,
+    create_replace_resource,
+)
+from ingestr.src.mailchimp.settings import (
+    MERGE_ENDPOINTS,
+    NESTED_ENDPOINTS,
+    REPLACE_ENDPOINTS,
+)
+
+
+@dlt.source(max_table_nesting=0, name="mailchimp_source")
+def mailchimp_source(
+    api_key: str,
+    server: str,
+) -> Iterable[DltResource]:
+    """
+    Mailchimp data source.
+
+    Args:
+        api_key: Mailchimp API key for authentication
+        server: Server prefix (e.g., 'us10')
+
+    Yields:
+        DltResource: Data resources for Mailchimp data
+    """
+    base_url = f"https://{server}.api.mailchimp.com/3.0"
+    session = create_client()
+    auth = ("anystring", api_key)
+
+    @dlt.resource(
+        name="account",
+        write_disposition="replace",
+    )
+    def fetch_account() -> Iterator[dict[str, Any]]:
+        """
+        Fetch account information from Mailchimp.
+
+        Table format: account (no parameters needed)
+        """
+        response = session.get(f"{base_url}/", auth=auth)
+        response.raise_for_status()
+        data = response.json()
+        yield data
+
+    # Create resources dynamically
+    resources = [fetch_account]
+
+    # Create merge resources (with incremental loading)
+    for (
+        resource_name,
+        endpoint_path,
+        data_key,
+        primary_key,
+        incremental_key,
+    ) in MERGE_ENDPOINTS:
+        resources.append(
+            create_merge_resource(
+                base_url,
+                session,
+                auth,
+                resource_name,
+                endpoint_path,
+                data_key,
+                primary_key,
+                incremental_key,
+            )
+        )
+
+    # Create replace resources (without incremental loading)
+    for replace_endpoint in REPLACE_ENDPOINTS:
+        resource_name, endpoint_path, data_key, pk = replace_endpoint
+        resources.append(
+            create_replace_resource(
+                base_url,
+                session,
+                auth,
+                resource_name,
+                endpoint_path,
+                data_key,
+                pk,
+            )
+        )
+
+    # Create nested resources (depend on parent resources)
+    for nested_endpoint in NESTED_ENDPOINTS:
+        (
+            parent_name,
+            parent_path,
+            parent_key,
+            parent_id_field,
+            nested_name,
+            nested_path,
+            nested_key,
+            pk,
+        ) = nested_endpoint
+        resources.append(
+            create_nested_resource(
+                base_url,
+                session,
+                auth,
+                parent_name,
+                parent_path,
+                parent_key,
+                parent_id_field,
+                nested_name,
+                nested_path,
+                nested_key,
+                pk,
+            )
+        )
+
+    return tuple(resources)

ingestr/src/mailchimp/helpers.py

@@ -0,0 +1,226 @@
+"""
+Helper functions for Mailchimp source.
+"""
+
+from typing import Any, Iterator
+
+import dlt
+
+
+def fetch_paginated(
+    session,
+    url: str,
+    auth: tuple,
+    data_key: str | None = None,
+) -> Iterator[dict[str, Any]]:
+    """
+    Helper function to fetch paginated data from Mailchimp API.
+
+    Args:
+        session: HTTP session
+        url: API endpoint URL
+        auth: Authentication tuple
+        data_key: Key in response containing the data array (if None, return whole response)
+
+    Yields:
+        Individual items from the paginated response
+    """
+    offset = 0
+    count = 1000  # Maximum allowed by Mailchimp
+
+    while True:
+        params = {"count": count, "offset": offset}
+        response = session.get(url, auth=auth, params=params)
+        response.raise_for_status()
+        data = response.json()
+
+        # Extract items from response
+        if data_key and data_key in data:
+            items = data[data_key]
+        elif isinstance(data, list):
+            items = data
+        else:
+            # If no data_key specified and response is dict, yield the whole response
+            yield data
+            break
+
+        if not items:
+            break
+
+        yield from items
+
+        # Check if we've received fewer items than requested (last page)
+        if len(items) < count:
+            break
+
+        offset += count
+
+
+def create_merge_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    name: str,
+    path: str,
+    key: str,
+    pk: str,
+    ik: str,
+):
+    """
+    Create a DLT resource with merge disposition for incremental loading.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        name: Resource name
+        path: API endpoint path
+        key: Data key in response
+        pk: Primary key field
+        ik: Incremental key field
+
+    Returns:
+        DLT resource function
+    """
+
+    @dlt.resource(
+        name=name,
+        write_disposition="merge",
+        primary_key=pk,
+    )
+    def fetch_data(
+        updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
+            ik, initial_value=None
+        ),
+    ) -> Iterator[dict[str, Any]]:
+        url = f"{base_url}/{path}"
+        yield from fetch_paginated(session, url, auth, data_key=key)
+
+    return fetch_data
+
+
+def create_replace_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    name: str,
+    path: str,
+    key: str,
+    pk: str | None,
+):
+    """
+    Create a DLT resource with replace disposition.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        name: Resource name
+        path: API endpoint path
+        key: Data key in response
+        pk: Primary key field (optional)
+
+    Returns:
+        DLT resource function
+    """
+
+    def fetch_data() -> Iterator[dict[str, Any]]:
+        url = f"{base_url}/{path}"
+        yield from fetch_paginated(session, url, auth, data_key=key)
+
+    # Apply the resource decorator with conditional primary_key
+    if pk is not None:
+        return dlt.resource(
+            fetch_data,
+            name=name,
+            write_disposition="replace",
+            primary_key=pk,
+        )
+    else:
+        return dlt.resource(
+            fetch_data,
+            name=name,
+            write_disposition="replace",
+        )
+
+
+def create_nested_resource(
+    base_url: str,
+    session,
+    auth: tuple,
+    parent_resource_name: str,
+    parent_path: str,
+    parent_key: str,
+    parent_id_field: str,
+    nested_name: str,
+    nested_path: str,
+    nested_key: str | None,
+    pk: str | None,
+):
+    """
+    Create a nested DLT resource that depends on a parent resource.
+
+    Args:
+        base_url: Base API URL
+        session: HTTP session
+        auth: Authentication tuple
+        parent_resource_name: Name of the parent resource
+        parent_path: Parent API endpoint path
+        parent_key: Data key in parent response
+        parent_id_field: Field name for parent ID
+        nested_name: Nested resource name
+        nested_path: Nested API endpoint path (with {id} placeholder)
+        nested_key: Data key in nested response (None to return whole response)
+        pk: Primary key field (optional)
+
+    Returns:
+        DLT resource function
+    """
+
+    def fetch_nested_data() -> Iterator[dict[str, Any]]:
+        # First, fetch parent items
+        parent_url = f"{base_url}/{parent_path}"
+        parent_items = fetch_paginated(session, parent_url, auth, data_key=parent_key)
+
+        # For each parent item, fetch nested data
+        for parent_item in parent_items:
+            parent_id = parent_item.get(parent_id_field)
+            if parent_id:
+                # Build nested URL with parent ID
+                nested_url = f"{base_url}/{nested_path.format(id=parent_id)}"
+
+                # Fetch nested data
+                response = session.get(nested_url, auth=auth)
+                response.raise_for_status()
+                data = response.json()
+
+                # Extract nested items or return whole response
+                if nested_key and nested_key in data:
+                    items = data[nested_key]
+                    if isinstance(items, list):
+                        for item in items:
+                            # Add parent reference
+                            item[f"{parent_resource_name}_id"] = parent_id
+                            yield item
+                    else:
+                        items[f"{parent_resource_name}_id"] = parent_id
+                        yield items
+                else:
+                    # Return whole response with parent reference
+                    data[f"{parent_resource_name}_id"] = parent_id
+                    yield data
+
+    # Apply the resource decorator with conditional primary_key
+    if pk is not None:
+        return dlt.resource(
+            fetch_nested_data,
+            name=nested_name,
+            write_disposition="replace",
+            primary_key=pk,
+        )
+    else:
+        return dlt.resource(
+            fetch_nested_data,
+            name=nested_name,
+            write_disposition="replace",
+        )

ingestr/src/mailchimp/settings.py

@@ -0,0 +1,164 @@
+"""
+Mailchimp API endpoint configurations.
+"""
+
+# Endpoints with merge disposition (have both primary_key and incremental_key)
+# Format: (resource_name, endpoint_path, data_key, primary_key, incremental_key)
+MERGE_ENDPOINTS = [
+    ("audiences", "lists", "lists", "id", "date_created"),
+    ("automations", "automations", "automations", "id", "create_time"),
+    ("campaigns", "campaigns", "campaigns", "id", "create_time"),
+    ("connected_sites", "connected-sites", "sites", "id", "updated_at"),
+    ("conversations", "conversations", "conversations", "id", "last_message.timestamp"),
+    ("ecommerce_stores", "ecommerce/stores", "stores", "id", "updated_at"),
+    ("facebook_ads", "facebook-ads", "facebook_ads", "id", "updated_at"),
+    ("landing_pages", "landing-pages", "landing_pages", "id", "updated_at"),
+    ("reports", "reports", "reports", "id", "send_time"),
+]
+
+# Endpoints with replace disposition
+# Format: (resource_name, endpoint_path, data_key, primary_key)
+REPLACE_ENDPOINTS: list[tuple[str, str, str, str | None]] = [
+    ("account_exports", "account-exports", "exports", None),
+    ("authorized_apps", "authorized-apps", "apps", "id"),
+    ("batches", "batches", "batches", None),
+    ("campaign_folders", "campaign-folders", "folders", "id"),
+    ("chimp_chatter", "activity-feed/chimp-chatter", "chimp_chatter", None),
+]
+
+# Nested endpoints (depend on parent resources)
+# Format: (parent_name, parent_path, parent_key, parent_id_field, nested_name, nested_path, nested_key, pk)
+NESTED_ENDPOINTS: list[tuple[str, str, str, str, str, str, str | None, str | None]] = [
+    # Reports nested endpoints
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_advice",
+        "reports/{id}/advice",
+        None,
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_domain_performance",
+        "reports/{id}/domain-performance",
+        "domains",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_locations",
+        "reports/{id}/locations",
+        "locations",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_sent_to",
+        "reports/{id}/sent-to",
+        "sent_to",
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_sub_reports",
+        "reports/{id}/sub-reports",
+        None,
+        None,
+    ),
+    (
+        "reports",
+        "reports",
+        "reports",
+        "id",
+        "reports_unsubscribed",
+        "reports/{id}/unsubscribed",
+        "unsubscribes",
+        None,
+    ),
+    # Lists/Audiences nested endpoints
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_activity",
+        "lists/{id}/activity",
+        "activity",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_clients",
+        "lists/{id}/clients",
+        "clients",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_growth_history",
+        "lists/{id}/growth-history",
+        "history",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_interest_categories",
+        "lists/{id}/interest-categories",
+        "categories",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_locations",
+        "lists/{id}/locations",
+        "locations",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_merge_fields",
+        "lists/{id}/merge-fields",
+        "merge_fields",
+        None,
+    ),
+    (
+        "audiences",
+        "lists",
+        "lists",
+        "id",
+        "lists_segments",
+        "lists/{id}/segments",
+        "segments",
+        None,
+    ),
+]

ingestr/src/sources.py

@@ -3971,3 +3971,29 @@ class MondaySource:
             ).with_resources(table_name)
         except ResourcesNotFoundError:
             raise UnsupportedResourceError(table_name, "Monday")
+
+
+class MailchimpSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        query_params = parse_qs(parsed_uri.query)
+        api_key = query_params.get("api_key")
+        server = query_params.get("server")
+
+        if api_key is None:
+            raise MissingValueError("api_key", "Mailchimp")
+        if server is None:
+            raise MissingValueError("server", "Mailchimp")
+
+        from ingestr.src.mailchimp import mailchimp_source
+
+        try:
+            return mailchimp_source(
+                api_key=api_key[0],
+                server=server[0],
+            ).with_resources(table)
+        except ResourcesNotFoundError:
+            raise UnsupportedResourceError(table, "Mailchimp")

{ingestr-0.14.9.dist-info → ingestr-0.14.91.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.9
+Version: 0.14.91
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.9.dist-info → ingestr-0.14.91.dist-info}/RECORD

@@ -2,17 +2,17 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
 ingestr/main.py,sha256=qo0g3wCFl8a_1jUwXagX8L1Q8PKKQlTF7md9pfnzW0Y,27155
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
-ingestr/src/buildinfo.py,sha256=JVeLatT3zF_GlPLBMGJdol2FzZDg-jgHFHrlQBdckG8,20
+ingestr/src/buildinfo.py,sha256=S1bddOwYtoZWcUt0-yZ-Xo-uAYE96jHPwDOFnYbIpyg,21
 ingestr/src/destinations.py,sha256=QtjE0AGs0WkPHaI2snWPHJ8HHi4lwXUBYLJPklz8Mvk,27772
 ingestr/src/errors.py,sha256=fhJ2BxOqOsBfOxuTDKfZblvawBrPG3x_1VikIxMZBRI,874
-ingestr/src/factory.py,sha256=03eGDe2rL6qyT5sGmTGZi-XIwJbbdoedE_KjW3ZF7QY,7661
+ingestr/src/factory.py,sha256=z6lcaGwK8o7AcaeshX7TP4WfB_jsIi0TvarcquGYnK0,7720
 ingestr/src/filters.py,sha256=0n0sNAVG_f-B_1r7lW5iNtw9z_G1bxWzPaiL1i6tnbU,1665
 ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,732
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
 ingestr/src/masking.py,sha256=VN0LdfvExhQ1bZMRylGtaBUIoH-vjuIUmRnYKwo3yiY,11358
 ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
 ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
-ingestr/src/sources.py,sha256=bHkR8wZ46TBgvIWxpBp_xm-KHBnSEYbjVDKtAOzAB2I,138819
+ingestr/src/sources.py,sha256=DSJAiwfVV0czMpHX1d93gQ7xIk_RyGPubQ31KfT5N8I,139639
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -109,6 +109,9 @@ ingestr/src/linear/helpers.py,sha256=J9lTuu8rHHM3YTA082_wfvByW6Teie4_44eYaVmDBhQ
 ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
 ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
 ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
+ingestr/src/mailchimp/__init__.py,sha256=-V4IacEJ8wvngHQsSuGu_Z4xAt1XtQIHigvOoWcndw0,3175
+ingestr/src/mailchimp/helpers.py,sha256=KXchokX5ApLqpjIAHzKWkI23ETGmQMlUrbOC5rm7q6k,6166
+ingestr/src/mailchimp/settings.py,sha256=0x9q-8tvieuDZb0H8GNdNWnRFmVDo2jK68IoBVdKE6k,4014
 ingestr/src/mixpanel/__init__.py,sha256=s1QtqMP0BTGW6YtdCabJFWj7lEn7KujzELwGpBOQgfs,1796
 ingestr/src/mixpanel/client.py,sha256=c_reouegOVYBOwHLfgYFwpmkba0Sxro1Zkml07NCYf0,3602
 ingestr/src/monday/__init__.py,sha256=ZNdGCC_1CEYlgxAef-5QO56Drm9IMP82-rZpEvbD8aY,6918
@@ -180,8 +183,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
 ingestr/tests/unit/test_smartsheets.py,sha256=zf3DXT29Y4TH2lNPBFphdjlaelUUyPJcsW2UO68RzDs,4862
-ingestr-0.14.9.dist-info/METADATA,sha256=xqQRDYjtnSD01hUyNk3jnTc-hjCfWd8CEsAPmx5XzUk,15265
-ingestr-0.14.9.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.14.9.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.14.9.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.14.9.dist-info/RECORD,,
+ingestr-0.14.91.dist-info/METADATA,sha256=S_ZjXBHtmd92ZS7l6u_Xg6W0TpXynfhXp7AA4WggEI4,15266
+ingestr-0.14.91.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.14.91.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.14.91.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.14.91.dist-info/RECORD,,