ingestr 0.13.21__py3-none-any.whl → 0.13.23__py3-none-any.whl

ingestr/src/adjust/adjust_helpers.py

@@ -82,7 +82,9 @@ class AdjustAPI:
             items = result.get("rows", [])
             yield items
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
+            raise HTTPError(
+                f"Request failed with status code: {response.status_code}, {response.text}."
+            )
 
     def fetch_events(self):
         headers = {"Authorization": f"Bearer {self.api_key}"}
@@ -93,7 +95,9 @@ class AdjustAPI:
             result = response.json()
             yield result
         else:
-            raise HTTPError(f"Request failed with status code: {response.status_code}, {response.text}.")
+            raise HTTPError(
+                f"Request failed with status code: {response.status_code}, {response.text}."
+            )
 
 
 def parse_filters(filters_raw: str) -> dict:

ingestr/src/applovin_max/__init__.py

@@ -105,11 +105,13 @@ def get_data(
         if response.status_code == 404:
             if "No Mediation App Id found for platform" in response.text:
                 return None
-        error_message = f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        error_message = (
+            f"AppLovin MAX API error (status {response.status_code}): {response.text}"
+        )
         raise requests.HTTPError(error_message)
-    
+
     response_url = response.json().get("ad_revenue_report_url")
     df = pd.read_csv(response_url)
     df["Date"] = pd.to_datetime(df["Date"])
     df["partition_date"] = df["Date"].dt.date
-    return df
+    return df

ingestr/src/buildinfo.py

@@ -1 +1 @@
-version = "v0.13.21"
+version = "v0.13.23"

ingestr/src/factory.py

@@ -43,6 +43,7 @@ from ingestr.src.sources import (
     MongoDbSource,
     NotionSource,
     PersonioSource,
+    PipedriveSource,
     S3Source,
     SalesforceSource,
     ShopifySource,
@@ -144,6 +145,7 @@ class SourceDestinationFactory:
         "salesforce": SalesforceSource,
         "personio": PersonioSource,
         "kinesis": KinesisSource,
+        "pipedrive": PipedriveSource,
     }
     destinations: Dict[str, Type[DestinationProtocol]] = {
         "bigquery": BigQueryDestination,

ingestr/src/hubspot/__init__.py

@@ -199,7 +199,6 @@ def crm_objects(
 
     props = ",".join(sorted(list(set(props))))
 
-
     params = {"properties": props, "limit": 100}
 
     yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, params=params)

ingestr/src/kinesis/__init__.py

@@ -16,7 +16,7 @@ from .helpers import get_shard_iterator, max_sequence_by_shard
     name=lambda args: args["stream_name"],
     primary_key="kinesis_msg_id",
     standalone=True,
-    max_table_nesting=0
+    max_table_nesting=0,
 )
 def kinesis_stream(
     stream_name: str,
@@ -75,7 +75,6 @@ def kinesis_stream(
 
     # get next shard to fetch messages from
     while shard_id := shard_ids.pop(0) if shard_ids else None:
-     
         shard_iterator, _ = get_shard_iterator(
             kinesis_client,
             stream_name,
@@ -83,14 +82,14 @@ def kinesis_stream(
             last_msg,  # type: ignore
             initial_at_datetime,  # type: ignore
         )
-       
+
         while shard_iterator:
             records = []
             records_response = kinesis_client.get_records(
                 ShardIterator=shard_iterator,
                 Limit=chunk_size,  # The size of data can be up to 1 MB, it must be controlled by the user
             )
-            
+
             for record in records_response["Records"]:
                 sequence_number = record["SequenceNumber"]
                 content = record["Data"]

ingestr/src/partition.py

@@ -13,7 +13,6 @@ def apply_athena_hints(
     additional_hints: Dict[str, TColumnSchema] = {},
 ) -> None:
     def _apply_partition_hint(resource: DltResource) -> None:
-
         columns = resource.columns if resource.columns else {}
 
         partition_hint = (
@@ -24,7 +23,8 @@ def apply_athena_hints(
         athena_adapter(
             resource,
             athena_partition.day(partition_column)
-            if partition_hint and partition_hint.get("data_type") in ("timestamp", "date")
+            if partition_hint
+            and partition_hint.get("data_type") in ("timestamp", "date")
             else partition_column,
         )
 

ingestr/src/pipedrive/__init__.py

@@ -0,0 +1,198 @@
+"""Highly customizable source for Pipedrive, supports endpoint addition, selection and column rename
+
+Pipedrive api docs: https://developers.pipedrive.com/docs/api/v1
+
+Pipedrive changes or deprecates fields and endpoints without versioning the api.
+If something breaks, it's a good idea to check the changelog.
+Api changelog: https://developers.pipedrive.com/changelog
+
+To get an api key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+"""
+
+from typing import Any, Dict, Iterator, List, Optional, Union  # noqa: F401
+
+import dlt
+from dlt.common import pendulum
+from dlt.common.time import ensure_pendulum_datetime
+from dlt.sources import DltResource, TDataItems
+
+from .helpers import group_deal_flows
+from .helpers.custom_fields_munger import rename_fields, update_fields_mapping
+from .helpers.pages import get_pages, get_recent_items_incremental
+from .settings import ENTITY_MAPPINGS, RECENTS_ENTITIES
+from .typing import TDataPage
+
+
+@dlt.source(name="pipedrive", max_table_nesting=0)
+def pipedrive_source(
+    pipedrive_api_key: str = dlt.secrets.value,
+    since_timestamp: Optional[Union[pendulum.DateTime, str]] = "1970-01-01 00:00:00",
+) -> Iterator[DltResource]:
+    """
+    Get data from the Pipedrive API. Supports incremental loading and custom fields mapping.
+
+    Args:
+        pipedrive_api_key: https://pipedrive.readme.io/docs/how-to-find-the-api-token
+        since_timestamp: Starting timestamp for incremental loading. By default complete history is loaded on first run.
+        incremental: Enable or disable incremental loading.
+
+    Returns resources:
+        custom_fields_mapping
+        activities
+        activityTypes
+        deals
+        deals_flow
+        deals_participants
+        files
+        filters
+        notes
+        persons
+        organizations
+        pipelines
+        products
+        stages
+        users
+        leads
+
+    For custom fields rename the `custom_fields_mapping` resource must be selected or loaded before other resources.
+
+    Resources that depend on another resource are implemented as transformers
+    so they can re-use the original resource data without re-downloading.
+    Examples:  deals_participants, deals_flow
+    """
+
+    # yield nice rename mapping
+    yield create_state(pipedrive_api_key) | parsed_mapping
+
+    # parse timestamp and build kwargs
+    since_timestamp = ensure_pendulum_datetime(since_timestamp).strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    resource_kwargs: Any = (
+        {"since_timestamp": since_timestamp} if since_timestamp else {}
+    )
+
+    # create resources for all endpoints
+    endpoints_resources = {}
+    for entity, resource_name in RECENTS_ENTITIES.items():
+        endpoints_resources[resource_name] = dlt.resource(
+            get_recent_items_incremental,
+            name=resource_name,
+            primary_key="id",
+            write_disposition="merge",
+        )(entity, pipedrive_api_key, **resource_kwargs)
+
+    yield from endpoints_resources.values()
+
+    # create transformers for deals to participants and flows
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_participants", write_disposition="merge", primary_key="id"
+    )(_get_deals_participants)(pipedrive_api_key)
+
+    yield endpoints_resources["deals"] | dlt.transformer(
+        name="deals_flow", write_disposition="merge", primary_key="id"
+    )(_get_deals_flow)(pipedrive_api_key)
+
+    yield leads(pipedrive_api_key, update_time=since_timestamp)
+
+
+def _get_deals_flow(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataItems]:
+    custom_fields_mapping = dlt.current.source_state().get("custom_fields_mapping", {})
+    for row in deals_page:
+        url = f"deals/{row['id']}/flow"
+        pages = get_pages(url, pipedrive_api_key)
+        for entity, page in group_deal_flows(pages):
+            yield dlt.mark.with_table_name(
+                rename_fields(page, custom_fields_mapping.get(entity, {})),
+                "deals_flow_" + entity,
+            )
+
+
+def _get_deals_participants(
+    deals_page: TDataPage, pipedrive_api_key: str
+) -> Iterator[TDataPage]:
+    for row in deals_page:
+        url = f"deals/{row['id']}/participants"
+        yield from get_pages(url, pipedrive_api_key)
+
+
+@dlt.resource(selected=False)
+def create_state(pipedrive_api_key: str) -> Iterator[Dict[str, Any]]:
+    def _get_pages_for_rename(
+        entity: str, fields_entity: str, pipedrive_api_key: str
+    ) -> Dict[str, Any]:
+        existing_fields_mapping: Dict[str, Dict[str, str]] = (
+            custom_fields_mapping.setdefault(entity, {})
+        )
+        # we need to process all pages before yielding
+        for page in get_pages(fields_entity, pipedrive_api_key):
+            existing_fields_mapping = update_fields_mapping(
+                page, existing_fields_mapping
+            )
+        return existing_fields_mapping
+
+    # gets all *Fields data and stores in state
+    custom_fields_mapping = dlt.current.source_state().setdefault(
+        "custom_fields_mapping", {}
+    )
+    for entity, fields_entity, _ in ENTITY_MAPPINGS:
+        if fields_entity is None:
+            continue
+        custom_fields_mapping[entity] = _get_pages_for_rename(
+            entity, fields_entity, pipedrive_api_key
+        )
+
+    yield custom_fields_mapping
+
+
+@dlt.transformer(
+    name="custom_fields_mapping",
+    write_disposition="replace",
+    columns={"options": {"data_type": "json"}},
+)
+def parsed_mapping(
+    custom_fields_mapping: Dict[str, Any],
+) -> Optional[Iterator[List[Dict[str, str]]]]:
+    """
+    Parses and yields custom fields' mapping in order to be stored in destiny by dlt
+    """
+    for endpoint, data_item_mapping in custom_fields_mapping.items():
+        yield [
+            {
+                "endpoint": endpoint,
+                "hash_string": hash_string,
+                "name": names["name"],
+                "normalized_name": names["normalized_name"],
+                "options": names["options"],
+                "field_type": names["field_type"],
+            }
+            for hash_string, names in data_item_mapping.items()
+        ]
+
+
+@dlt.resource(primary_key="id", write_disposition="merge")
+def leads(
+    pipedrive_api_key: str = dlt.secrets.value,
+    update_time: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "update_time", "1970-01-01 00:00:00"
+    ),
+) -> Iterator[TDataPage]:
+    """Resource to incrementally load pipedrive leads by update_time"""
+    # Leads inherit custom fields from deals
+    fields_mapping = (
+        dlt.current.source_state().get("custom_fields_mapping", {}).get("deals", {})
+    )
+    # Load leads pages sorted from newest to oldest and stop loading when
+    # last incremental value is reached
+    pages = get_pages(
+        "leads",
+        pipedrive_api_key,
+        extra_params={"sort": "update_time DESC"},
+    )
+    for page in pages:
+        yield rename_fields(page, fields_mapping)
+
+        if update_time.start_out_of_range:
+            return

ingestr/src/pipedrive/helpers/__init__.py

@@ -0,0 +1,23 @@
+"""Pipedrive source helpers"""
+
+from itertools import groupby
+from typing import Any, Dict, Iterable, List, Tuple, cast  # noqa: F401
+
+from dlt.common import pendulum  # noqa: F401
+
+
+def _deals_flow_group_key(item: Dict[str, Any]) -> str:
+    return item["object"]  # type: ignore[no-any-return]
+
+
+def group_deal_flows(
+    pages: Iterable[Iterable[Dict[str, Any]]],
+) -> Iterable[Tuple[str, List[Dict[str, Any]]]]:
+    for page in pages:
+        for entity, items in groupby(
+            sorted(page, key=_deals_flow_group_key), key=_deals_flow_group_key
+        ):
+            yield (
+                entity,
+                [dict(item["data"], timestamp=item["timestamp"]) for item in items],
+            )

ingestr/src/pipedrive/helpers/custom_fields_munger.py

@@ -0,0 +1,102 @@
+from typing import Any, Dict, Optional, TypedDict
+
+import dlt
+
+from ..typing import TDataPage
+
+
+class TFieldMapping(TypedDict):
+    name: str
+    normalized_name: str
+    options: Optional[Dict[str, str]]
+    field_type: str
+
+
+def update_fields_mapping(
+    new_fields_mapping: TDataPage, existing_fields_mapping: Dict[str, Any]
+) -> Dict[str, Any]:
+    """
+    Specific function to perform data munging and push changes to custom fields' mapping stored in dlt's state
+    The endpoint must be an entity fields' endpoint
+    """
+    for data_item in new_fields_mapping:
+        # 'edit_flag' field contains a boolean value, which is set to 'True' for custom fields and 'False' otherwise.
+        if data_item.get("edit_flag"):
+            # Regarding custom fields, 'key' field contains pipedrive's hash string representation of its name
+            # We assume that pipedrive's hash strings are meant to be an univoque representation of custom fields' name, so dlt's state shouldn't be updated while those values
+            # remain unchanged
+            existing_fields_mapping = _update_field(data_item, existing_fields_mapping)
+        # Built in enum and set fields are mapped if their options have int ids
+        # Enum fields with bool and string key options are left intact
+        elif data_item.get("field_type") in {"set", "enum"}:
+            options = data_item.get("options", [])
+            first_option = options[0]["id"] if len(options) >= 1 else None
+            if isinstance(first_option, int) and not isinstance(first_option, bool):
+                existing_fields_mapping = _update_field(
+                    data_item, existing_fields_mapping
+                )
+    return existing_fields_mapping
+
+
+def _update_field(
+    data_item: Dict[str, Any],
+    existing_fields_mapping: Optional[Dict[str, TFieldMapping]],
+) -> Dict[str, TFieldMapping]:
+    """Create or update the given field's info the custom fields state
+    If the field hash already exists in the state from previous runs the name is not updated.
+    New enum options (if any) are appended to the state.
+    """
+    existing_fields_mapping = existing_fields_mapping or {}
+    key = data_item["key"]
+    options = data_item.get("options", [])
+    new_options_map = {str(o["id"]): o["label"] for o in options}
+    existing_field = existing_fields_mapping.get(key)
+    if not existing_field:
+        existing_fields_mapping[key] = dict(
+            name=data_item["name"],
+            normalized_name=_normalized_name(data_item["name"]),
+            options=new_options_map,
+            field_type=data_item["field_type"],
+        )
+        return existing_fields_mapping
+    existing_options = existing_field.get("options", {})
+    if not existing_options or existing_options == new_options_map:
+        existing_field["options"] = new_options_map
+        existing_field["field_type"] = data_item[
+            "field_type"
+        ]  # Add for backwards compat
+        return existing_fields_mapping
+    # Add new enum options to the existing options array
+    # so that when option is renamed the original label remains valid
+    new_option_keys = set(new_options_map) - set(existing_options)
+    for key in new_option_keys:
+        existing_options[key] = new_options_map[key]
+    existing_field["options"] = existing_options
+    return existing_fields_mapping
+
+
+def _normalized_name(name: str) -> str:
+    source_schema = dlt.current.source_schema()
+    normalized_name = name.strip()  # remove leading and trailing spaces
+    return source_schema.naming.normalize_identifier(normalized_name)
+
+
+def rename_fields(data: TDataPage, fields_mapping: Dict[str, Any]) -> TDataPage:
+    if not fields_mapping:
+        return data
+    for data_item in data:
+        for hash_string, field in fields_mapping.items():
+            if hash_string not in data_item:
+                continue
+            field_value = data_item.pop(hash_string)
+            field_name = field["name"]
+            options_map = field["options"]
+            # Get label instead of ID for 'enum' and 'set' fields
+            if field_value and field["field_type"] == "set":  # Multiple choice
+                field_value = [
+                    options_map.get(str(enum_id), enum_id) for enum_id in field_value
+                ]
+            elif field_value and field["field_type"] == "enum":
+                field_value = options_map.get(str(field_value), field_value)
+            data_item[field_name] = field_value
+    return data

ingestr/src/pipedrive/helpers/pages.py

@@ -0,0 +1,115 @@
+from itertools import chain
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    Iterator,
+    List,
+    TypeVar,
+    Union,
+)
+
+import dlt
+from dlt.sources.helpers import requests
+
+from ..typing import TDataPage
+from .custom_fields_munger import rename_fields
+
+
+def get_pages(
+    entity: str, pipedrive_api_key: str, extra_params: Dict[str, Any] = None
+) -> Iterator[List[Dict[str, Any]]]:
+    """
+    Generic method to retrieve endpoint data based on the required headers and params.
+
+    Args:
+        entity: the endpoint you want to call
+        pipedrive_api_key:
+        extra_params: any needed request params except pagination.
+
+    Returns:
+
+    """
+    headers = {"Content-Type": "application/json"}
+    params = {"api_token": pipedrive_api_key}
+    if extra_params:
+        params.update(extra_params)
+    url = f"https://app.pipedrive.com/v1/{entity}"
+    yield from _paginated_get(url, headers=headers, params=params)
+
+
+def get_recent_items_incremental(
+    entity: str,
+    pipedrive_api_key: str,
+    since_timestamp: dlt.sources.incremental[str] = dlt.sources.incremental(
+        "update_time|modified", "1970-01-01 00:00:00"
+    ),
+) -> Iterator[TDataPage]:
+    """Get a specific entity type from /recents with incremental state."""
+    yield from _get_recent_pages(entity, pipedrive_api_key, since_timestamp.last_value)
+
+
+def _paginated_get(
+    url: str, headers: Dict[str, Any], params: Dict[str, Any]
+) -> Iterator[List[Dict[str, Any]]]:
+    """
+    Requests and yields data 500 records at a time
+    Documentation: https://pipedrive.readme.io/docs/core-api-concepts-pagination
+    """
+    # pagination start and page limit
+    params["start"] = 0
+    params["limit"] = 500
+    while True:
+        page = requests.get(url, headers=headers, params=params).json()
+        # yield data only
+        data = page["data"]
+        if data:
+            yield data
+        # check if next page exists
+        pagination_info = page.get("additional_data", {}).get("pagination", {})
+        # is_next_page is set to True or False
+        if not pagination_info.get("more_items_in_collection", False):
+            break
+        params["start"] = pagination_info.get("next_start")
+
+
+T = TypeVar("T")
+
+
+def _extract_recents_data(data: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Results from recents endpoint contain `data` key which is either a single entity or list of entities
+
+    This returns a flat list of entities from an iterable of recent results
+    """
+    return [
+        data_item
+        for data_item in chain.from_iterable(
+            (_list_wrapped(item["data"]) for item in data)
+        )
+        if data_item is not None
+    ]
+
+
+def _list_wrapped(item: Union[List[T], T]) -> List[T]:
+    if isinstance(item, list):
+        return item
+    return [item]
+
+
+def _get_recent_pages(
+    entity: str, pipedrive_api_key: str, since_timestamp: str
+) -> Iterator[TDataPage]:
+    custom_fields_mapping = (
+        dlt.current.source_state().get("custom_fields_mapping", {}).get(entity, {})
+    )
+    pages = get_pages(
+        "recents",
+        pipedrive_api_key,
+        extra_params=dict(since_timestamp=since_timestamp, items=entity),
+    )
+    pages = (_extract_recents_data(page) for page in pages)
+    for page in pages:
+        yield rename_fields(page, custom_fields_mapping)
+
+
+__source_name__ = "pipedrive"

ingestr/src/pipedrive/settings.py

@@ -0,0 +1,27 @@
+"""Pipedrive source settings and constants"""
+
+ENTITY_MAPPINGS = [
+    ("activity", "activityFields", {"user_id": 0}),
+    ("organization", "organizationFields", None),
+    ("person", "personFields", None),
+    ("product", "productFields", None),
+    ("deal", "dealFields", None),
+    ("pipeline", None, None),
+    ("stage", None, None),
+    ("user", None, None),
+]
+
+RECENTS_ENTITIES = {
+    "activity": "activities",
+    "activityType": "activity_types",
+    "deal": "deals",
+    "file": "files",
+    "filter": "filters",
+    "note": "notes",
+    "person": "persons",
+    "organization": "organizations",
+    "pipeline": "pipelines",
+    "product": "products",
+    "stage": "stages",
+    "user": "users",
+}

ingestr/src/pipedrive/typing.py

@@ -0,0 +1,3 @@
+from typing import Any, Dict, List
+
+TDataPage = List[Dict[str, Any]]

ingestr/src/sources.py

@@ -85,6 +85,7 @@ from ingestr.src.linkedin_ads.dimension_time_enum import (
 from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.personio import personio_source
+from ingestr.src.pipedrive import pipedrive_source
 from ingestr.src.salesforce import salesforce_source
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
@@ -177,7 +178,7 @@ class SqlSource:
                 scheme="clickhouse+native",
                 query=urlencode(query_params, doseq=True),
             ).geturl()
-         
+
         if uri.startswith("db2://"):
             uri = uri.replace("db2://", "db2+ibm_db://")
 
@@ -1837,8 +1838,8 @@ class AppLovinSource:
 
 
 class ApplovinMaxSource:
-    #expected uri format: applovinmax://?api_key=<api_key>
-    #expected table format: user_ad_revenue:app_id_1,app_id_2
+    # expected uri format: applovinmax://?api_key=<api_key>
+    # expected table format: user_ad_revenue:app_id_1,app_id_2
 
     def handles_incrementality(self) -> bool:
         return True
@@ -1850,7 +1851,7 @@ class ApplovinMaxSource:
         api_key = params.get("api_key")
         if api_key is None:
             raise ValueError("api_key is required to connect to AppLovin Max API.")
-        
+
         AVAILABLE_TABLES = ["user_ad_revenue"]
 
         table_fields = table.split(":")
@@ -1860,7 +1861,7 @@ class ApplovinMaxSource:
             raise ValueError(
                 "Invalid table format. Expected format is user_ad_revenue:app_id_1,app_id_2"
             )
-        
+
         if requested_table not in AVAILABLE_TABLES:
             raise ValueError(
                 f"Table name '{requested_table}' is not supported for AppLovin Max source yet."
@@ -1868,17 +1869,15 @@ class ApplovinMaxSource:
                 "If you need additional tables, please create a GitHub issue at "
                 "https://github.com/bruin-data/ingestr"
             )
-        
-        applications = [i for i in table_fields[1].replace(" ", "").split(",") if i.strip()]
+
+        applications = [
+            i for i in table_fields[1].replace(" ", "").split(",") if i.strip()
+        ]
         if len(applications) == 0:
-            raise ValueError(
-                "At least one application id is required"
-            )
-    
+            raise ValueError("At least one application id is required")
+
         if len(applications) != len(set(applications)):
-            raise ValueError(
-                "Application ids must be unique."
-            )
+            raise ValueError("Application ids must be unique.")
 
         interval_start = kwargs.get("interval_start")
         interval_end = kwargs.get("interval_end")
@@ -2009,3 +2008,36 @@ class KinesisSource:
         return kinesis_stream(
             stream_name=table, credentials=credentials, initial_at_timestamp=start_date
         )
+
+
+class PipedriveSource:
+    def handles_incrementality(self) -> bool:
+        return True
+
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        parsed_uri = urlparse(uri)
+        params = parse_qs(parsed_uri.query)
+        api_key = params.get("api_token")
+        if api_key is None:
+            raise MissingValueError("api_token", "Pipedrive")
+
+        start_date = kwargs.get("interval_start")
+        if start_date is not None:
+            start_date = ensure_pendulum_datetime(start_date)
+        else:
+            start_date = pendulum.parse("2000-01-01")
+
+        if table not in [
+            "users",
+            "activities",
+            "persons",
+            "organizations",
+            "products",
+            "stages",
+            "deals",
+        ]:
+            raise UnsupportedResourceError(table, "Pipedrive")
+
+        return pipedrive_source(
+            pipedrive_api_key=api_key, since_timestamp=start_date
+        ).with_resources(table)

{ingestr-0.13.21.dist-info → ingestr-0.13.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.21
+Version: 0.13.23
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -16,7 +16,7 @@ Classifier: Topic :: Database
 Requires-Python: >=3.9
 Requires-Dist: aiobotocore==2.21.1
 Requires-Dist: aiohappyeyeballs==2.4.8
-Requires-Dist: aiohttp==3.11.13
+Requires-Dist: aiohttp==3.11.15
 Requires-Dist: aioitertools==0.12.0
 Requires-Dist: aiosignal==1.3.2
 Requires-Dist: alembic==1.15.1
@@ -55,8 +55,8 @@ Requires-Dist: facebook-business==20.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
-Requires-Dist: fsspec==2024.10.0
-Requires-Dist: gcsfs==2024.10.0
+Requires-Dist: fsspec==2025.3.2
+Requires-Dist: gcsfs==2025.3.2
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -149,7 +149,7 @@ Requires-Dist: rich-argparse==1.7.0
 Requires-Dist: rich==13.9.4
 Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
-Requires-Dist: s3fs==2024.10.0
+Requires-Dist: s3fs==2025.3.2
 Requires-Dist: s3transfer==0.11.3
 Requires-Dist: scramp==1.4.5
 Requires-Dist: semver==3.0.4

{ingestr-0.13.21.dist-info → ingestr-0.13.23.dist-info}/RECORD

@@ -1,23 +1,23 @@
 ingestr/main.py,sha256=74lbiWEa27MUKFPbyUNGIlrwD5fRxej5cKFwe_LX1pE,25452
 ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
 ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
-ingestr/src/buildinfo.py,sha256=zDreaGxFwbbedppZHtPlS8zOtO8gysfQdbEx3DXHebA,21
+ingestr/src/buildinfo.py,sha256=0fhX5lBsaw-iKqJ6ErFYyjAyJLE9JWuIUG2T45nhhM8,21
 ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
 ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
-ingestr/src/factory.py,sha256=-Mv9vWvY7PJOnOSMvLYq6zLtPLYpTORLD6bUvf8xUc8,5209
+ingestr/src/factory.py,sha256=1jqcLv_QUUGeyg1OYN3ywrRdcDZyDRtMOongwyjDapU,5268
 ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
 ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
-ingestr/src/partition.py,sha256=ZqcCTz6xrSurgJDSZYn-_gaBaEXj4peV7N1wYe3IiQk,953
+ingestr/src/partition.py,sha256=E0WHqh1FTheQAIVK_-jWUx0dgyYZCD1VxlAm362gao4,964
 ingestr/src/resource.py,sha256=XG-sbBapFVEM7OhHQFQRTdTLlh-mHB-N4V1t8F8Tsww,543
-ingestr/src/sources.py,sha256=7WL0QVfQJNwyGLCJzIa_m3d7z-eA3etp-klIErtGvco,71081
+ingestr/src/sources.py,sha256=kGsgFWf8Ghha0-HlC6PlDIIKX2Lriah4UmAseziGdr4,72035
 ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
 ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
 ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
 ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
-ingestr/src/adjust/adjust_helpers.py,sha256=8PbDTmO3dnPrVGBA1qCmkJamVZOLN52oE1gkSsfDeZM,3682
+ingestr/src/adjust/adjust_helpers.py,sha256=G_EvNuvA7CsaOtbV3g249iAyggMDMZYbtWOzOAz_EjY,3742
 ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
 ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
-ingestr/src/applovin_max/__init__.py,sha256=o0aL4jBZqwK528MVw9dS1G5EZbF4tx6_Ef0IfqkhAT0,3294
+ingestr/src/applovin_max/__init__.py,sha256=CBMADQ23gi0oxAsxe-RV67GGb8I4EFOX_It45Vv9dj4,3315
 ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
 ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
 ingestr/src/appstore/__init__.py,sha256=3P4VZH2WJF477QjW19jMTwu6L8DXcLkYSdutnvp3AmM,4742
@@ -58,12 +58,12 @@ ingestr/src/google_sheets/helpers/api_calls.py,sha256=RiVfdacbaneszhmuhYilkJnkc9
 ingestr/src/google_sheets/helpers/data_processing.py,sha256=WYO6z4XjGcG0Hat2J2enb-eLX5mSNVb2vaqRE83FBWU,11000
 ingestr/src/gorgias/__init__.py,sha256=_mFkMYwlY5OKEY0o_FK1OKol03A-8uk7bm1cKlmt5cs,21432
 ingestr/src/gorgias/helpers.py,sha256=DamuijnvhGY9hysQO4txrVMf4izkGbh5qfBKImdOINE,5427
-ingestr/src/hubspot/__init__.py,sha256=jDI9PIgz2cvRMbTtX2tF1FcATFpBOMI7M8Ua4G7qDCc,9465
+ingestr/src/hubspot/__init__.py,sha256=NYgSIAPXQh2Qp1eKun7TgcerKogq6pWtNkr-_f0FXbI,9464
 ingestr/src/hubspot/helpers.py,sha256=PTn-UHJv1ENIvA5azUTaHCmFXgmHLJC1tUatQ1N-KFE,6727
 ingestr/src/hubspot/settings.py,sha256=9P1OKiRL88kl_m8n1HhuG-Qpq9VGbqPLn5Q0QYneToU,2193
 ingestr/src/kafka/__init__.py,sha256=wMCXdiraeKd1Kssi9WcVCGZaNGm2tJEtnNyuB4aR5_k,3541
 ingestr/src/kafka/helpers.py,sha256=V9WcVn3PKnEpggArHda4vnAcaV8VDuh__dSmRviJb5Y,7502
-ingestr/src/kinesis/__init__.py,sha256=6Sqg8VQp0yhz8xCm1LPvciOUT8SrD6u5Klmfu7p4Y8c,6229
+ingestr/src/kinesis/__init__.py,sha256=u5ThH1y8uObZKXgIo71em1UnX6MsVHWOjcf1jKqKbE8,6205
 ingestr/src/kinesis/helpers.py,sha256=aF0GCDKSectaaW8XPdERY_6bUs0ky19dcBs24ZFn-o0,2473
 ingestr/src/klaviyo/_init_.py,sha256=ucWHqBe8DQvXVpbmxKFAV5ljpCFb4ps_2QTD0OSiWxY,7905
 ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
@@ -80,6 +80,12 @@ ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falP
 ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
 ingestr/src/personio/__init__.py,sha256=sHYpoV-rg-kA1YsflctChis0hKcTrL6mka9O0CHV4zA,11638
 ingestr/src/personio/helpers.py,sha256=EKmBN0Lf4R0lc3yqqs7D-RjoZ75E8gPcctt59xwHxrY,2901
+ingestr/src/pipedrive/__init__.py,sha256=iRrxeMwo8_83ptgGnTFTNHV1nYvIsFfg0a3XzugPYeI,6982
+ingestr/src/pipedrive/settings.py,sha256=q119Fy4C5Ip1rMoCILX2BkHV3bwiXC_dW58KIiDUzsY,708
+ingestr/src/pipedrive/typing.py,sha256=lEMXu4hhAA3XkhVSlBUa-juqyupisd3c-qSQKxFvzoE,69
+ingestr/src/pipedrive/helpers/__init__.py,sha256=UX1K_qnGXB0ShtnBOfp2XuVbK8RRoCK8TdEmIjRckgg,710
+ingestr/src/pipedrive/helpers/custom_fields_munger.py,sha256=rZ4AjdITHfJE2NNomCR7vMBS1KnWpEGVF6fADwsIHUE,4488
+ingestr/src/pipedrive/helpers/pages.py,sha256=Klpjw2OnMuhzit3PpiHKsfzGcJ3rQPSQBl3HhE3-6eA,3358
 ingestr/src/salesforce/__init__.py,sha256=2hik5pRrxVODdDTlUEMoyccNC07zozjnxkMHcjMT1qA,4558
 ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
 ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
@@ -112,8 +118,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
 ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
 ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
 ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
-ingestr-0.13.21.dist-info/METADATA,sha256=0OnGAlj9MYS-U-GBMiwPwLwd7d9jGP99dynFKboc1P8,13630
-ingestr-0.13.21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-ingestr-0.13.21.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
-ingestr-0.13.21.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
-ingestr-0.13.21.dist-info/RECORD,,
+ingestr-0.13.23.dist-info/METADATA,sha256=-ZhEaYU1LxDj565cfpYl75R5JS-G-HFL8KtLygNR22Q,13627
+ingestr-0.13.23.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+ingestr-0.13.23.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
+ingestr-0.13.23.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
+ingestr-0.13.23.dist-info/RECORD,,