omniload 0.0.0.dev0__py3-none-any.whl

omniload/src/pipedrive/__init__.py

@@ -0,0 +1,212 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Highly customizable source for Pipedrive, supports endpoint addition, selection and column rename
+
+Pipedrive api docs: https://developers.pipedrive.com/docs/api/v1
+
+Pipedrive changes or deprecates fields and endpoints without versioning the api.
+If something breaks, it's a good idea to check the changelog.
+Api changelog: https://developers.pipedrive.com/changelog
+
+To get an api key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+"""
+
+from typing import Any, Dict, Iterator, List, Optional, Union  # noqa: F401
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource, TDataItems
+
+from .helpers import group_deal_flows
+from .helpers.custom_fields_munger import rename_fields, update_fields_mapping
+from .helpers.pages import get_pages, get_recent_items_incremental
+from .settings import ENTITY_MAPPINGS, RECENTS_ENTITIES
+from .typing import TDataPage
+
+
+@dlt.source(name="pipedrive", max_table_nesting=0)
+def pipedrive_source(
+    pipedrive_api_key: str = dlt.secrets.value,
+    since_timestamp: Optional[Union[pendulum.DateTime, str]] = "1970-01-01 00:00:00",
+) -> Iterator[DltResource]:
+    """
+    Get data from the Pipedrive API. Supports incremental loading and custom fields mapping.
+
+    Args:
+        pipedrive_api_key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+        since_timestamp: Starting timestamp for incremental loading. By default complete history is loaded on first run.
+        incremental: Enable or disable incremental loading.
+
+    Returns resources:
+        custom_fields_mapping
+        activities
+        activityTypes
+        deals
+        deals_flow
+        deals_participants
+        files
+        filters
+        notes
+        persons
+        organizations
+        pipelines
+        products
+        stages
+        users
+        leads
+
+    For custom fields rename the `custom_fields_mapping` resource must be selected or loaded before other resources.
+
+    Resources that depend on another resource are implemented as transformers
+    so they can re-use the original resource data without re-downloading.
+    Examples:  deals_participants, deals_flow
+    """
+
+    # yield nice rename mapping
+    yield create_state(pipedrive_api_key) | parsed_mapping
+
+    # parse timestamp and build kwargs
+    since_timestamp = ensure_pendulum_datetime(since_timestamp).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    resource_kwargs: Any = (
+        {"since_timestamp": since_timestamp} if since_timestamp else {}
+    )
+
+    # create resources for all endpoints
+    endpoints_resources = {}
+    for entity, resource_name in RECENTS_ENTITIES.items():
+        endpoints_resources[resource_name] = dlt.resource(
+            get_recent_items_incremental,
+            name=resource_name,
+            primary_key="id",
+            write_disposition="merge",
+        )(entity, pipedrive_api_key, **resource_kwargs)
+
+    yield from endpoints_resources.values()
+
+    # create transformers for deals to participants and flows
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_participants", write_disposition="merge", primary_key="id"
+    )(_get_deals_participants)(pipedrive_api_key)
+
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_flow", write_disposition="merge", primary_key="id"
+    )(_get_deals_flow)(pipedrive_api_key)
+
+    yield leads(pipedrive_api_key, update_time=since_timestamp)
+
+
+def _get_deals_flow(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataItems]:
+    custom_fields_mapping = dlt.current.source_state().get("custom_fields_mapping", {})
+    for row in deals_page:
+        url = f"deals/{row['id']}/flow"
+        pages = get_pages(url, pipedrive_api_key)
+        for entity, page in group_deal_flows(pages):
+            yield dlt.mark.with_table_name(
+                rename_fields(page, custom_fields_mapping.get(entity, {})),
+                "deals_flow_" + entity,
+            )
+
+
+def _get_deals_participants(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataPage]:
+    for row in deals_page:
+        url = f"deals/{row['id']}/participants"
+        yield from get_pages(url, pipedrive_api_key)
+
+
+@dlt.resource(selected=False)
+def create_state(pipedrive_api_key: str) -> Iterator[Dict[str, Any]]:
+    def _get_pages_for_rename(
+        entity: str, fields_entity: str, pipedrive_api_key: str
+    ) -> Dict[str, Any]:
+        existing_fields_mapping: Dict[str, Dict[str, str]] = (
+            custom_fields_mapping.setdefault(entity, {})
+        )
+        # we need to process all pages before yielding
+        for page in get_pages(fields_entity, pipedrive_api_key):
+            existing_fields_mapping = update_fields_mapping(
+                page, existing_fields_mapping
+            )
+        return existing_fields_mapping
+
+    # gets all *Fields data and stores in state
+    custom_fields_mapping = dlt.current.source_state().setdefault(
+        "custom_fields_mapping", {}
+    )
+    for entity, fields_entity, _ in ENTITY_MAPPINGS:
+        if fields_entity is None:
+            continue
+        custom_fields_mapping[entity] = _get_pages_for_rename(
+            entity, fields_entity, pipedrive_api_key
+        )
+
+    yield custom_fields_mapping
+
+
+@dlt.transformer(
+    name="custom_fields_mapping",
+    write_disposition="replace",
+    columns={"options": {"data_type": "json"}},
+)
+def parsed_mapping(
+    custom_fields_mapping: Dict[str, Any],
+) -> Optional[Iterator[List[Dict[str, str]]]]:
+    """
+    Parses and yields custom fields' mapping in order to be stored in destiny by dlt
+    """
+    for endpoint, data_item_mapping in custom_fields_mapping.items():
+        yield [
+            {
+                "endpoint": endpoint,
+                "hash_string": hash_string,
+                "name": names["name"],
+                "normalized_name": names["normalized_name"],
+                "options": names["options"],
+                "field_type": names["field_type"],
+            }
+            for hash_string, names in data_item_mapping.items()
+        ]
+
+
+@dlt.resource(primary_key="id", write_disposition="merge")
+def leads(
+    pipedrive_api_key: str = dlt.secrets.value,
+    update_time: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "update_time", "1970-01-01 00:00:00"
+    ),
+) -> Iterator[TDataPage]:
+    """Resource to incrementally load pipedrive leads by update_time"""
+    # Leads inherit custom fields from deals
+    fields_mapping = (
+        dlt.current.source_state().get("custom_fields_mapping", {}).get("deals", {})
+    )
+    # Load leads pages sorted from newest to oldest and stop loading when
+    # last incremental value is reached
+    pages = get_pages(
+        "leads",
+        pipedrive_api_key,
+        extra_params={"sort": "update_time DESC"},
+    )
+    for page in pages:
+        yield rename_fields(page, fields_mapping)
+
+        if update_time.start_out_of_range:
+            return

omniload/src/pipedrive/helpers/__init__.py

@@ -0,0 +1,37 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pipedrive source helpers"""
+
+from itertools import groupby
+from typing import Any, Dict, Iterable, List, Tuple, cast  # noqa: F401
+
+from dlt.common import pendulum  # noqa: F401
+
+
+def _deals_flow_group_key(item: Dict[str, Any]) -> str:
+    return item["object"]  # type: ignore[no-any-return]
+
+
+def group_deal_flows(
+    pages: Iterable[Iterable[Dict[str, Any]]],
+) -> Iterable[Tuple[str, List[Dict[str, Any]]]]:
+    for page in pages:
+        for entity, items in groupby(
+            sorted(page, key=_deals_flow_group_key), key=_deals_flow_group_key
+        ):
+            yield (
+                entity,
+                [dict(item["data"], timestamp=item["timestamp"]) for item in items],
+            )

omniload/src/pipedrive/helpers/custom_fields_munger.py

@@ -0,0 +1,116 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, Optional, TypedDict
+
+import dlt
+
+from ..typing import TDataPage
+
+
+class TFieldMapping(TypedDict):
+    name: str
+    normalized_name: str
+    options: Optional[Dict[str, str]]
+    field_type: str
+
+
+def update_fields_mapping(
+    new_fields_mapping: TDataPage, existing_fields_mapping: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Specific function to perform data munging and push changes to custom fields' mapping stored in dlt's state
+    The endpoint must be an entity fields' endpoint
+    """
+    for data_item in new_fields_mapping:
+        # 'edit_flag' field contains a boolean value, which is set to 'True' for custom fields and 'False' otherwise.
+        if data_item.get("edit_flag"):
+            # Regarding custom fields, 'key' field contains pipedrive's hash string representation of its name
+            # We assume that pipedrive's hash strings are meant to be an univoque representation of custom fields' name, so dlt's state shouldn't be updated while those values
+            # remain unchanged
+            existing_fields_mapping = _update_field(data_item, existing_fields_mapping)
+        # Built in enum and set fields are mapped if their options have int ids
+        # Enum fields with bool and string key options are left intact
+        elif data_item.get("field_type") in {"set", "enum"}:
+            options = data_item.get("options", [])
+            first_option = options[0]["id"] if len(options) >= 1 else None
+            if isinstance(first_option, int) and not isinstance(first_option, bool):
+                existing_fields_mapping = _update_field(
+                    data_item, existing_fields_mapping
+                )
+    return existing_fields_mapping
+
+
+def _update_field(
+    data_item: Dict[str, Any],
+    existing_fields_mapping: Optional[Dict[str, TFieldMapping]],
+) -> Dict[str, TFieldMapping]:
+    """Create or update the given field's info the custom fields state
+    If the field hash already exists in the state from previous runs the name is not updated.
+    New enum options (if any) are appended to the state.
+    """
+    existing_fields_mapping = existing_fields_mapping or {}
+    key = data_item["key"]
+    options = data_item.get("options", [])
+    new_options_map = {str(o["id"]): o["label"] for o in options}
+    existing_field = existing_fields_mapping.get(key)
+    if not existing_field:
+        existing_fields_mapping[key] = dict(
+            name=data_item["name"],
+            normalized_name=_normalized_name(data_item["name"]),
+            options=new_options_map,
+            field_type=data_item["field_type"],
+        )
+        return existing_fields_mapping
+    existing_options = existing_field.get("options", {})
+    if not existing_options or existing_options == new_options_map:
+        existing_field["options"] = new_options_map
+        existing_field["field_type"] = data_item[
+            "field_type"
+        ]  # Add for backwards compat
+        return existing_fields_mapping
+    # Add new enum options to the existing options array
+    # so that when option is renamed the original label remains valid
+    new_option_keys = set(new_options_map) - set(existing_options)
+    for key in new_option_keys:
+        existing_options[key] = new_options_map[key]
+    existing_field["options"] = existing_options
+    return existing_fields_mapping
+
+
+def _normalized_name(name: str) -> str:
+    source_schema = dlt.current.source_schema()
+    normalized_name = name.strip()  # remove leading and trailing spaces
+    return source_schema.naming.normalize_identifier(normalized_name)
+
+
+def rename_fields(data: TDataPage, fields_mapping: Dict[str, Any]) -> TDataPage:
+    if not fields_mapping:
+        return data
+    for data_item in data:
+        for hash_string, field in fields_mapping.items():
+            if hash_string not in data_item:
+                continue
+            field_value = data_item.pop(hash_string)
+            field_name = field["name"]
+            options_map = field["options"]
+            # Get label instead of ID for 'enum' and 'set' fields
+            if field_value and field["field_type"] == "set":  # Multiple choice
+                field_value = [
+                    options_map.get(str(enum_id), enum_id) for enum_id in field_value
+                ]
+            elif field_value and field["field_type"] == "enum":
+                field_value = options_map.get(str(field_value), field_value)
+            data_item[field_name] = field_value
+    return data

omniload/src/pipedrive/helpers/pages.py

@@ -0,0 +1,129 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from itertools import chain
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    TypeVar,
+    Union,
+)
+
+import dlt
+from dlt.sources.helpers import requests
+
+from ..typing import TDataPage
+from .custom_fields_munger import rename_fields
+
+
+def get_pages(
+    entity: str, pipedrive_api_key: str, extra_params: Dict[str, Any] = None
+) -> Iterator[List[Dict[str, Any]]]:
+    """
+    Generic method to retrieve endpoint data based on the required headers and params.
+
+    Args:
+        entity: the endpoint you want to call
+        pipedrive_api_key:
+        extra_params: any needed request params except pagination.
+
+    Returns:
+
+    """
+    headers = {"Content-Type": "application/json"}
+    params = {"api_token": pipedrive_api_key}
+    if extra_params:
+        params.update(extra_params)
+    url = f"https://app.pipedrive.com/v1/{entity}"
+    yield from _paginated_get(url, headers=headers, params=params)
+
+
+def get_recent_items_incremental(
+    entity: str,
+    pipedrive_api_key: str,
+    since_timestamp: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "update_time|modified", "1970-01-01 00:00:00"
+    ),
+) -> Iterator[TDataPage]:
+    """Get a specific entity type from /recents with incremental state."""
+    yield from _get_recent_pages(entity, pipedrive_api_key, since_timestamp.last_value)
+
+
+def _paginated_get(
+    url: str, headers: Dict[str, Any], params: Dict[str, Any]
+) -> Iterator[List[Dict[str, Any]]]:
+    """
+    Requests and yields data 500 records at a time
+    Documentation: https://pipedrive.readme.io/docs/core-api-concepts-pagination
+    """
+    # pagination start and page limit
+    params["start"] = 0
+    params["limit"] = 500
+    while True:
+        page = requests.get(url, headers=headers, params=params).json()
+        # yield data only
+        data = page["data"]
+        if data:
+            yield data
+        # check if next page exists
+        pagination_info = page.get("additional_data", {}).get("pagination", {})
+        # is_next_page is set to True or False
+        if not pagination_info.get("more_items_in_collection", False):
+            break
+        params["start"] = pagination_info.get("next_start")
+
+
+T = TypeVar("T")
+
+
+def _extract_recents_data(data: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Results from recents endpoint contain `data` key which is either a single entity or list of entities
+
+    This returns a flat list of entities from an iterable of recent results
+    """
+    return [
+        data_item
+        for data_item in chain.from_iterable(
+            (_list_wrapped(item["data"]) for item in data)
+        )
+        if data_item is not None
+    ]
+
+
+def _list_wrapped(item: Union[List[T], T]) -> List[T]:
+    if isinstance(item, list):
+        return item
+    return [item]
+
+
+def _get_recent_pages(
+    entity: str, pipedrive_api_key: str, since_timestamp: str
+) -> Iterator[TDataPage]:
+    custom_fields_mapping = (
+        dlt.current.source_state().get("custom_fields_mapping", {}).get(entity, {})
+    )
+    pages = get_pages(
+        "recents",
+        pipedrive_api_key,
+        extra_params=dict(since_timestamp=since_timestamp, items=entity),
+    )
+    pages = (_extract_recents_data(page) for page in pages)
+    for page in pages:
+        yield rename_fields(page, custom_fields_mapping)
+
+
+__source_name__ = "pipedrive"

omniload/src/pipedrive/settings.py

@@ -0,0 +1,41 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pipedrive source settings and constants"""
+
+ENTITY_MAPPINGS = [
+    ("activity", "activityFields", {"user_id": 0}),
+    ("organization", "organizationFields", None),
+    ("person", "personFields", None),
+    ("product", "productFields", None),
+    ("deal", "dealFields", None),
+    ("pipeline", None, None),
+    ("stage", None, None),
+    ("user", None, None),
+]
+
+RECENTS_ENTITIES = {
+    "activity": "activities",
+    "activityType": "activity_types",
+    "deal": "deals",
+    "file": "files",
+    "filter": "filters",
+    "note": "notes",
+    "person": "persons",
+    "organization": "organizations",
+    "pipeline": "pipelines",
+    "product": "products",
+    "stage": "stages",
+    "user": "users",
+}

omniload/src/pipedrive/typing.py

@@ -0,0 +1,17 @@
+# Copyright 2022-2025 ScaleVector
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+
+TDataPage = List[Dict[str, Any]]