ingestr 0.13.13__py3-none-any.whl → 0.14.104__py3-none-any.whl

ingestr/src/partition.py

@@ -0,0 +1,32 @@
+from typing import Dict
+
+from dlt.common.schema.typing import TColumnSchema
+from dlt.sources import DltResource, DltSource
+
+import ingestr.src.resource as resource
+
+
+def apply_athena_hints(
+    source: DltSource | DltResource,
+    partition_column: str,
+    additional_hints: Dict[str, TColumnSchema] = {},
+) -> None:
+    from dlt.destinations.adapters import athena_adapter, athena_partition
+
+    def _apply_partition_hint(resource: DltResource) -> None:
+        columns = resource.columns if resource.columns else {}
+
+        partition_hint = (
+            columns.get(partition_column)  # type: ignore
+            or additional_hints.get(partition_column)
+        )
+
+        athena_adapter(
+            resource,
+            athena_partition.day(partition_column)
+            if partition_hint
+            and partition_hint.get("data_type") in ("timestamp", "date")
+            else partition_column,
+        )
+
+    resource.for_each(source, _apply_partition_hint)

ingestr/src/phantombuster/__init__.py

@@ -0,0 +1,65 @@
+from typing import Iterable, Optional
+
+import dlt
+import pendulum
+import requests
+from dlt.common.typing import TAnyDateTime, TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers.requests import Client
+
+from ingestr.src.phantombuster.client import PhantombusterClient
+
+
+def retry_on_limit(
+    response: Optional[requests.Response], exception: Optional[BaseException]
+) -> bool:
+    if response is not None and response.status_code == 429:
+        return True
+    return False
+
+
+def create_client() -> requests.Session:
+    return Client(
+        raise_for_status=False,
+        retry_condition=retry_on_limit,
+        request_max_attempts=12,
+        request_backoff_factor=2,
+    ).session
+
+
+@dlt.source(max_table_nesting=0)
+def phantombuster_source(
+    api_key: str, agent_id: str, start_date: TAnyDateTime, end_date: TAnyDateTime | None
+) -> Iterable[DltResource]:
+    client = PhantombusterClient(api_key)
+
+    @dlt.resource(
+        write_disposition="merge",
+        primary_key="container_id",
+        columns={
+            "partition_dt": {"data_type": "date", "partition": True},
+        },
+    )
+    def completed_phantoms(
+        dateTime=(
+            dlt.sources.incremental(
+                "ended_at",
+                initial_value=start_date,
+                end_value=end_date,
+                range_start="closed",
+                range_end="closed",
+            )
+        ),
+    ) -> Iterable[TDataItem]:
+        if dateTime.end_value is None:
+            end_dt = pendulum.now(tz="UTC")
+        else:
+            end_dt = dateTime.end_value
+
+        start_dt = dateTime.last_value
+
+        yield client.fetch_containers_result(
+            create_client(), agent_id, start_date=start_dt, end_date=end_dt
+        )
+
+    return completed_phantoms

ingestr/src/phantombuster/client.py

@@ -0,0 +1,87 @@
+from typing import Union
+
+import pendulum
+import requests
+
+
+class PhantombusterClient:
+    def __init__(self, api_key: str):
+        self.api_key = api_key
+
+    def _get_headers(self):
+        return {
+            "X-Phantombuster-Key-1": self.api_key,
+            "accept": "application/json",
+        }
+
+    def fetch_containers_result(
+        self,
+        session: requests.Session,
+        agent_id: str,
+        start_date: pendulum.DateTime,
+        end_date: pendulum.DateTime,
+    ):
+        url = "https://api.phantombuster.com/api/v2/containers/fetch-all/"
+        before_ended_at = None
+        limit = 100
+
+        started_at = start_date.int_timestamp * 1000 + int(
+            start_date.microsecond / 1000
+        )
+        ended_at = end_date.int_timestamp * 1000 + int(end_date.microsecond / 1000)
+
+        while True:
+            params: dict[str, Union[str, int, float, bytes, None]] = {
+                "agentId": agent_id,
+                "limit": limit,
+                "mode": "finalized",
+            }
+
+            if before_ended_at:
+                params["beforeEndedAt"] = before_ended_at
+
+            response = session.get(url=url, headers=self._get_headers(), params=params)
+            data = response.json()
+            containers = data.get("containers", [])
+
+            for container in containers:
+                container_ended_at = container.get("endedAt")
+
+                if before_ended_at is None or before_ended_at > container_ended_at:
+                    before_ended_at = container_ended_at
+
+                if container_ended_at < started_at or container_ended_at > ended_at:
+                    continue
+
+                try:
+                    result = self.fetch_result_object(session, container["id"])
+                    partition_dt = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    ).date()
+                    container_ended_at_datetime = pendulum.from_timestamp(
+                        container_ended_at / 1000, tz="UTC"
+                    )
+                    row = {
+                        "container_id": container["id"],
+                        "container": container,
+                        "result": result,
+                        "partition_dt": partition_dt,
+                        "ended_at": container_ended_at_datetime,
+                    }
+                    yield row
+
+                except requests.RequestException as e:
+                    print(f"Error fetching result for container {container['id']}: {e}")
+
+            if data["maxLimitReached"] is False:
+                break
+
+    def fetch_result_object(self, session: requests.Session, container_id: str):
+        result_url = (
+            "https://api.phantombuster.com/api/v2/containers/fetch-result-object"
+        )
+        params = {"id": container_id}
+        response = session.get(result_url, headers=self._get_headers(), params=params)
+        response.raise_for_status()
+
+        return response.json()

ingestr/src/pinterest/__init__.py

@@ -0,0 +1,82 @@
+from typing import Iterable
+
+import dlt
+import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.common.typing import TDataItem
+from dlt.sources import DltResource
+from dlt.sources.helpers import requests
+
+
+@dlt.source(name="pinterest", max_table_nesting=0)
+def pinterest_source(
+    start_date: pendulum.DateTime,
+    access_token: str,
+    page_size: int = 200,
+    end_date: pendulum.DateTime | None = None,
+) -> Iterable[DltResource]:
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {access_token}"})
+    base_url = "https://api.pinterest.com/v5"
+
+    def fetch_data(
+        endpoint: str,
+        start_dt: pendulum.DateTime,
+        end_dt: pendulum.DateTime,
+    ) -> Iterable[TDataItem]:
+        url = f"{base_url}/{endpoint}"
+        params = {"page_size": page_size}
+        bookmark = None
+        while True:
+            if bookmark:
+                params["bookmark"] = bookmark
+
+            resp = session.get(url, params=params)
+            resp.raise_for_status()
+            data = resp.json()
+            items = data.get("items") or []
+
+            for item in items:
+                item_created = ensure_pendulum_datetime(item["created_at"])
+                if item_created <= start_dt:
+                    continue
+                if item_created > end_dt:
+                    continue
+                item["created_at"] = item_created
+                yield item
+
+            bookmark = data.get("bookmark")
+            if not bookmark:
+                break
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def pins(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("pins", _start_date, _end_date)
+
+    @dlt.resource(write_disposition="merge", primary_key="id")
+    def boards(
+        datetime=dlt.sources.incremental(
+            "created_at",
+            initial_value=start_date,
+            end_value=end_date,
+        ),
+    ) -> Iterable[TDataItem]:
+        _start_date = datetime.last_value or start_date
+        if end_date is None:
+            _end_date = pendulum.now("UTC")
+        else:
+            _end_date = datetime.end_value
+        yield from fetch_data("boards", _start_date, _end_date)
+
+    return pins, boards

ingestr/src/pipedrive/__init__.py

@@ -0,0 +1,198 @@
+"""Highly customizable source for Pipedrive, supports endpoint addition, selection and column rename
+
+Pipedrive api docs: https://developers.pipedrive.com/docs/api/v1
+
+Pipedrive changes or deprecates fields and endpoints without versioning the api.
+If something breaks, it's a good idea to check the changelog.
+Api changelog: https://developers.pipedrive.com/changelog
+
+To get an api key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+"""
+
+from typing import Any, Dict, Iterator, List, Optional, Union  # noqa: F401
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource, TDataItems
+
+from .helpers import group_deal_flows
+from .helpers.custom_fields_munger import rename_fields, update_fields_mapping
+from .helpers.pages import get_pages, get_recent_items_incremental
+from .settings import ENTITY_MAPPINGS, RECENTS_ENTITIES
+from .typing import TDataPage
+
+
+@dlt.source(name="pipedrive", max_table_nesting=0)
+def pipedrive_source(
+    pipedrive_api_key: str = dlt.secrets.value,
+    since_timestamp: Optional[Union[pendulum.DateTime, str]] = "1970-01-01 00:00:00",
+) -> Iterator[DltResource]:
+    """
+    Get data from the Pipedrive API. Supports incremental loading and custom fields mapping.
+
+    Args:
+        pipedrive_api_key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+        since_timestamp: Starting timestamp for incremental loading. By default complete history is loaded on first run.
+        incremental: Enable or disable incremental loading.
+
+    Returns resources:
+        custom_fields_mapping
+        activities
+        activityTypes
+        deals
+        deals_flow
+        deals_participants
+        files
+        filters
+        notes
+        persons
+        organizations
+        pipelines
+        products
+        stages
+        users
+        leads
+
+    For custom fields rename the `custom_fields_mapping` resource must be selected or loaded before other resources.
+
+    Resources that depend on another resource are implemented as transformers
+    so they can re-use the original resource data without re-downloading.
+    Examples:  deals_participants, deals_flow
+    """
+
+    # yield nice rename mapping
+    yield create_state(pipedrive_api_key) | parsed_mapping
+
+    # parse timestamp and build kwargs
+    since_timestamp = ensure_pendulum_datetime(since_timestamp).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    resource_kwargs: Any = (
+        {"since_timestamp": since_timestamp} if since_timestamp else {}
+    )
+
+    # create resources for all endpoints
+    endpoints_resources = {}
+    for entity, resource_name in RECENTS_ENTITIES.items():
+        endpoints_resources[resource_name] = dlt.resource(
+            get_recent_items_incremental,
+            name=resource_name,
+            primary_key="id",
+            write_disposition="merge",
+        )(entity, pipedrive_api_key, **resource_kwargs)
+
+    yield from endpoints_resources.values()
+
+    # create transformers for deals to participants and flows
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_participants", write_disposition="merge", primary_key="id"
+    )(_get_deals_participants)(pipedrive_api_key)
+
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_flow", write_disposition="merge", primary_key="id"
+    )(_get_deals_flow)(pipedrive_api_key)
+
+    yield leads(pipedrive_api_key, update_time=since_timestamp)
+
+
+def _get_deals_flow(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataItems]:
+    custom_fields_mapping = dlt.current.source_state().get("custom_fields_mapping", {})
+    for row in deals_page:
+        url = f"deals/{row['id']}/flow"
+        pages = get_pages(url, pipedrive_api_key)
+        for entity, page in group_deal_flows(pages):
+            yield dlt.mark.with_table_name(
+                rename_fields(page, custom_fields_mapping.get(entity, {})),
+                "deals_flow_" + entity,
+            )
+
+
+def _get_deals_participants(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataPage]:
+    for row in deals_page:
+        url = f"deals/{row['id']}/participants"
+        yield from get_pages(url, pipedrive_api_key)
+
+
+@dlt.resource(selected=False)
+def create_state(pipedrive_api_key: str) -> Iterator[Dict[str, Any]]:
+    def _get_pages_for_rename(
+        entity: str, fields_entity: str, pipedrive_api_key: str
+    ) -> Dict[str, Any]:
+        existing_fields_mapping: Dict[str, Dict[str, str]] = (
+            custom_fields_mapping.setdefault(entity, {})
+        )
+        # we need to process all pages before yielding
+        for page in get_pages(fields_entity, pipedrive_api_key):
+            existing_fields_mapping = update_fields_mapping(
+                page, existing_fields_mapping
+            )
+        return existing_fields_mapping
+
+    # gets all *Fields data and stores in state
+    custom_fields_mapping = dlt.current.source_state().setdefault(
+        "custom_fields_mapping", {}
+    )
+    for entity, fields_entity, _ in ENTITY_MAPPINGS:
+        if fields_entity is None:
+            continue
+        custom_fields_mapping[entity] = _get_pages_for_rename(
+            entity, fields_entity, pipedrive_api_key
+        )
+
+    yield custom_fields_mapping
+
+
+@dlt.transformer(
+    name="custom_fields_mapping",
+    write_disposition="replace",
+    columns={"options": {"data_type": "json"}},
+)
+def parsed_mapping(
+    custom_fields_mapping: Dict[str, Any],
+) -> Optional[Iterator[List[Dict[str, str]]]]:
+    """
+    Parses and yields custom fields' mapping in order to be stored in destiny by dlt
+    """
+    for endpoint, data_item_mapping in custom_fields_mapping.items():
+        yield [
+            {
+                "endpoint": endpoint,
+                "hash_string": hash_string,
+                "name": names["name"],
+                "normalized_name": names["normalized_name"],
+                "options": names["options"],
+                "field_type": names["field_type"],
+            }
+            for hash_string, names in data_item_mapping.items()
+        ]
+
+
+@dlt.resource(primary_key="id", write_disposition="merge")
+def leads(
+    pipedrive_api_key: str = dlt.secrets.value,
+    update_time: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "update_time", "1970-01-01 00:00:00"
+    ),
+) -> Iterator[TDataPage]:
+    """Resource to incrementally load pipedrive leads by update_time"""
+    # Leads inherit custom fields from deals
+    fields_mapping = (
+        dlt.current.source_state().get("custom_fields_mapping", {}).get("deals", {})
+    )
+    # Load leads pages sorted from newest to oldest and stop loading when
+    # last incremental value is reached
+    pages = get_pages(
+        "leads",
+        pipedrive_api_key,
+        extra_params={"sort": "update_time DESC"},
+    )
+    for page in pages:
+        yield rename_fields(page, fields_mapping)
+
+        if update_time.start_out_of_range:
+            return

ingestr/src/pipedrive/helpers/__init__.py

@@ -0,0 +1,23 @@
+"""Pipedrive source helpers"""
+
+from itertools import groupby
+from typing import Any, Dict, Iterable, List, Tuple, cast  # noqa: F401
+
+from dlt.common import pendulum  # noqa: F401
+
+
+def _deals_flow_group_key(item: Dict[str, Any]) -> str:
+    return item["object"]  # type: ignore[no-any-return]
+
+
+def group_deal_flows(
+    pages: Iterable[Iterable[Dict[str, Any]]],
+) -> Iterable[Tuple[str, List[Dict[str, Any]]]]:
+    for page in pages:
+        for entity, items in groupby(
+            sorted(page, key=_deals_flow_group_key), key=_deals_flow_group_key
+        ):
+            yield (
+                entity,
+                [dict(item["data"], timestamp=item["timestamp"]) for item in items],
+            )

ingestr/src/pipedrive/helpers/custom_fields_munger.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, Optional, TypedDict
+
+import dlt
+
+from ..typing import TDataPage
+
+
+class TFieldMapping(TypedDict):
+    name: str
+    normalized_name: str
+    options: Optional[Dict[str, str]]
+    field_type: str
+
+
+def update_fields_mapping(
+    new_fields_mapping: TDataPage, existing_fields_mapping: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Specific function to perform data munging and push changes to custom fields' mapping stored in dlt's state
+    The endpoint must be an entity fields' endpoint
+    """
+    for data_item in new_fields_mapping:
+        # 'edit_flag' field contains a boolean value, which is set to 'True' for custom fields and 'False' otherwise.
+        if data_item.get("edit_flag"):
+            # Regarding custom fields, 'key' field contains pipedrive's hash string representation of its name
+            # We assume that pipedrive's hash strings are meant to be an univoque representation of custom fields' name, so dlt's state shouldn't be updated while those values
+            # remain unchanged
+            existing_fields_mapping = _update_field(data_item, existing_fields_mapping)
+        # Built in enum and set fields are mapped if their options have int ids
+        # Enum fields with bool and string key options are left intact
+        elif data_item.get("field_type") in {"set", "enum"}:
+            options = data_item.get("options", [])
+            first_option = options[0]["id"] if len(options) >= 1 else None
+            if isinstance(first_option, int) and not isinstance(first_option, bool):
+                existing_fields_mapping = _update_field(
+                    data_item, existing_fields_mapping
+                )
+    return existing_fields_mapping
+
+
+def _update_field(
+    data_item: Dict[str, Any],
+    existing_fields_mapping: Optional[Dict[str, TFieldMapping]],
+) -> Dict[str, TFieldMapping]:
+    """Create or update the given field's info the custom fields state
+    If the field hash already exists in the state from previous runs the name is not updated.
+    New enum options (if any) are appended to the state.
+    """
+    existing_fields_mapping = existing_fields_mapping or {}
+    key = data_item["key"]
+    options = data_item.get("options", [])
+    new_options_map = {str(o["id"]): o["label"] for o in options}
+    existing_field = existing_fields_mapping.get(key)
+    if not existing_field:
+        existing_fields_mapping[key] = dict(
+            name=data_item["name"],
+            normalized_name=_normalized_name(data_item["name"]),
+            options=new_options_map,
+            field_type=data_item["field_type"],
+        )
+        return existing_fields_mapping
+    existing_options = existing_field.get("options", {})
+    if not existing_options or existing_options == new_options_map:
+        existing_field["options"] = new_options_map
+        existing_field["field_type"] = data_item[
+            "field_type"
+        ]  # Add for backwards compat
+        return existing_fields_mapping
+    # Add new enum options to the existing options array
+    # so that when option is renamed the original label remains valid
+    new_option_keys = set(new_options_map) - set(existing_options)
+    for key in new_option_keys:
+        existing_options[key] = new_options_map[key]
+    existing_field["options"] = existing_options
+    return existing_fields_mapping
+
+
+def _normalized_name(name: str) -> str:
+    source_schema = dlt.current.source_schema()
+    normalized_name = name.strip()  # remove leading and trailing spaces
+    return source_schema.naming.normalize_identifier(normalized_name)
+
+
+def rename_fields(data: TDataPage, fields_mapping: Dict[str, Any]) -> TDataPage:
+    if not fields_mapping:
+        return data
+    for data_item in data:
+        for hash_string, field in fields_mapping.items():
+            if hash_string not in data_item:
+                continue
+            field_value = data_item.pop(hash_string)
+            field_name = field["name"]
+            options_map = field["options"]
+            # Get label instead of ID for 'enum' and 'set' fields
+            if field_value and field["field_type"] == "set":  # Multiple choice
+                field_value = [
+                    options_map.get(str(enum_id), enum_id) for enum_id in field_value
+                ]
+            elif field_value and field["field_type"] == "enum":
+                field_value = options_map.get(str(field_value), field_value)
+            data_item[field_name] = field_value
+    return data