omniload 0.0.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omniload/conftest.py +72 -0
- omniload/main.py +810 -0
- omniload/src/.gitignore +10 -0
- omniload/src/adjust/__init__.py +108 -0
- omniload/src/adjust/adjust_helpers.py +122 -0
- omniload/src/airtable/__init__.py +84 -0
- omniload/src/allium/__init__.py +128 -0
- omniload/src/anthropic/__init__.py +277 -0
- omniload/src/anthropic/helpers.py +525 -0
- omniload/src/applovin/__init__.py +316 -0
- omniload/src/applovin_max/__init__.py +117 -0
- omniload/src/appsflyer/__init__.py +325 -0
- omniload/src/appsflyer/client.py +110 -0
- omniload/src/appstore/__init__.py +142 -0
- omniload/src/appstore/client.py +126 -0
- omniload/src/appstore/errors.py +15 -0
- omniload/src/appstore/models.py +117 -0
- omniload/src/appstore/resources.py +179 -0
- omniload/src/arrow/__init__.py +81 -0
- omniload/src/asana_source/__init__.py +281 -0
- omniload/src/asana_source/helpers.py +30 -0
- omniload/src/asana_source/settings.py +158 -0
- omniload/src/attio/__init__.py +102 -0
- omniload/src/attio/helpers.py +65 -0
- omniload/src/blob.py +95 -0
- omniload/src/bruin/__init__.py +76 -0
- omniload/src/chess/__init__.py +180 -0
- omniload/src/chess/helpers.py +35 -0
- omniload/src/chess/settings.py +18 -0
- omniload/src/clickup/__init__.py +85 -0
- omniload/src/clickup/helpers.py +47 -0
- omniload/src/collector/spinner.py +43 -0
- omniload/src/couchbase_source/__init__.py +118 -0
- omniload/src/couchbase_source/helpers.py +135 -0
- omniload/src/cursor/__init__.py +83 -0
- omniload/src/cursor/helpers.py +188 -0
- omniload/src/customer_io/__init__.py +486 -0
- omniload/src/customer_io/helpers.py +530 -0
- omniload/src/destinations.py +982 -0
- omniload/src/docebo/__init__.py +589 -0
- omniload/src/docebo/client.py +435 -0
- omniload/src/docebo/helpers.py +97 -0
- omniload/src/dune/__init__.py +104 -0
- omniload/src/dune/helpers.py +108 -0
- omniload/src/dynamodb/__init__.py +86 -0
- omniload/src/elasticsearch/__init__.py +80 -0
- omniload/src/elasticsearch/helpers.py +141 -0
- omniload/src/errors.py +26 -0
- omniload/src/facebook_ads/__init__.py +403 -0
- omniload/src/facebook_ads/exceptions.py +19 -0
- omniload/src/facebook_ads/helpers.py +296 -0
- omniload/src/facebook_ads/settings.py +224 -0
- omniload/src/facebook_ads/utils.py +53 -0
- omniload/src/factory.py +305 -0
- omniload/src/filesystem/__init__.py +133 -0
- omniload/src/filesystem/helpers.py +114 -0
- omniload/src/filesystem/readers.py +187 -0
- omniload/src/filters.py +62 -0
- omniload/src/fireflies/__init__.py +151 -0
- omniload/src/fireflies/helpers.py +753 -0
- omniload/src/fluxx/__init__.py +10013 -0
- omniload/src/fluxx/helpers.py +233 -0
- omniload/src/frankfurter/__init__.py +157 -0
- omniload/src/frankfurter/helpers.py +48 -0
- omniload/src/freshdesk/__init__.py +103 -0
- omniload/src/freshdesk/freshdesk_client.py +151 -0
- omniload/src/freshdesk/settings.py +23 -0
- omniload/src/fundraiseup/__init__.py +95 -0
- omniload/src/fundraiseup/client.py +81 -0
- omniload/src/github/__init__.py +202 -0
- omniload/src/github/helpers.py +207 -0
- omniload/src/github/queries.py +129 -0
- omniload/src/github/settings.py +24 -0
- omniload/src/google_ads/__init__.py +198 -0
- omniload/src/google_ads/field.py +17 -0
- omniload/src/google_ads/metrics.py +254 -0
- omniload/src/google_ads/predicates.py +37 -0
- omniload/src/google_ads/reports.py +411 -0
- omniload/src/google_ads/test_google_ads.py +184 -0
- omniload/src/google_analytics/__init__.py +144 -0
- omniload/src/google_analytics/helpers.py +312 -0
- omniload/src/google_sheets/README.md +95 -0
- omniload/src/google_sheets/__init__.py +166 -0
- omniload/src/google_sheets/helpers/__init__.py +15 -0
- omniload/src/google_sheets/helpers/api_calls.py +160 -0
- omniload/src/google_sheets/helpers/data_processing.py +316 -0
- omniload/src/gorgias/__init__.py +595 -0
- omniload/src/gorgias/helpers.py +166 -0
- omniload/src/hostaway/__init__.py +302 -0
- omniload/src/hostaway/client.py +288 -0
- omniload/src/http/__init__.py +38 -0
- omniload/src/http/readers.py +146 -0
- omniload/src/http_client.py +24 -0
- omniload/src/hubspot/__init__.py +800 -0
- omniload/src/hubspot/helpers.py +417 -0
- omniload/src/hubspot/settings.py +329 -0
- omniload/src/indeed/__init__.py +153 -0
- omniload/src/indeed/helpers.py +228 -0
- omniload/src/influxdb/__init__.py +46 -0
- omniload/src/influxdb/client.py +34 -0
- omniload/src/intercom/__init__.py +142 -0
- omniload/src/intercom/helpers.py +674 -0
- omniload/src/intercom/settings.py +279 -0
- omniload/src/isoc_pulse/__init__.py +159 -0
- omniload/src/jira_source/__init__.py +377 -0
- omniload/src/jira_source/helpers.py +510 -0
- omniload/src/jira_source/settings.py +184 -0
- omniload/src/kafka/__init__.py +120 -0
- omniload/src/kafka/helpers.py +241 -0
- omniload/src/kinesis/__init__.py +153 -0
- omniload/src/kinesis/helpers.py +96 -0
- omniload/src/klaviyo/__init__.py +237 -0
- omniload/src/klaviyo/client.py +212 -0
- omniload/src/klaviyo/helpers.py +19 -0
- omniload/src/linear/__init__.py +634 -0
- omniload/src/linear/helpers.py +111 -0
- omniload/src/linkedin_ads/__init__.py +266 -0
- omniload/src/linkedin_ads/dimension_time_enum.py +17 -0
- omniload/src/linkedin_ads/helpers.py +246 -0
- omniload/src/loader.py +69 -0
- omniload/src/mailchimp/__init__.py +126 -0
- omniload/src/mailchimp/helpers.py +226 -0
- omniload/src/mailchimp/settings.py +164 -0
- omniload/src/masking.py +344 -0
- omniload/src/mixpanel/__init__.py +62 -0
- omniload/src/mixpanel/client.py +104 -0
- omniload/src/monday/__init__.py +246 -0
- omniload/src/monday/helpers.py +392 -0
- omniload/src/monday/settings.py +325 -0
- omniload/src/mongodb/__init__.py +281 -0
- omniload/src/mongodb/helpers.py +975 -0
- omniload/src/notion/__init__.py +69 -0
- omniload/src/notion/helpers/__init__.py +14 -0
- omniload/src/notion/helpers/client.py +178 -0
- omniload/src/notion/helpers/database.py +92 -0
- omniload/src/notion/settings.py +17 -0
- omniload/src/partition.py +32 -0
- omniload/src/personio/__init__.py +345 -0
- omniload/src/personio/helpers.py +100 -0
- omniload/src/phantombuster/__init__.py +65 -0
- omniload/src/phantombuster/client.py +87 -0
- omniload/src/pinterest/__init__.py +82 -0
- omniload/src/pipedrive/__init__.py +212 -0
- omniload/src/pipedrive/helpers/__init__.py +37 -0
- omniload/src/pipedrive/helpers/custom_fields_munger.py +116 -0
- omniload/src/pipedrive/helpers/pages.py +129 -0
- omniload/src/pipedrive/settings.py +41 -0
- omniload/src/pipedrive/typing.py +17 -0
- omniload/src/plusvibeai/__init__.py +335 -0
- omniload/src/plusvibeai/helpers.py +544 -0
- omniload/src/plusvibeai/settings.py +252 -0
- omniload/src/primer/__init__.py +45 -0
- omniload/src/primer/helpers.py +79 -0
- omniload/src/quickbooks/__init__.py +117 -0
- omniload/src/reddit_ads/__init__.py +183 -0
- omniload/src/reddit_ads/helpers.py +232 -0
- omniload/src/resource.py +40 -0
- omniload/src/revenuecat/__init__.py +83 -0
- omniload/src/revenuecat/helpers.py +237 -0
- omniload/src/salesforce/__init__.py +170 -0
- omniload/src/salesforce/helpers.py +78 -0
- omniload/src/shopify/__init__.py +1953 -0
- omniload/src/shopify/exceptions.py +17 -0
- omniload/src/shopify/helpers.py +202 -0
- omniload/src/shopify/settings.py +19 -0
- omniload/src/slack/__init__.py +290 -0
- omniload/src/slack/helpers.py +218 -0
- omniload/src/slack/settings.py +36 -0
- omniload/src/smartsheets/__init__.py +82 -0
- omniload/src/snapchat_ads/__init__.py +455 -0
- omniload/src/snapchat_ads/client.py +72 -0
- omniload/src/snapchat_ads/helpers.py +630 -0
- omniload/src/snapchat_ads/settings.py +130 -0
- omniload/src/socrata_source/__init__.py +83 -0
- omniload/src/socrata_source/helpers.py +85 -0
- omniload/src/socrata_source/settings.py +8 -0
- omniload/src/solidgate/__init__.py +219 -0
- omniload/src/solidgate/helpers.py +154 -0
- omniload/src/sources.py +5408 -0
- omniload/src/sql_database/__init__.py +0 -0
- omniload/src/sql_database/callbacks.py +66 -0
- omniload/src/stripe_analytics/__init__.py +183 -0
- omniload/src/stripe_analytics/helpers.py +386 -0
- omniload/src/stripe_analytics/settings.py +80 -0
- omniload/src/table_definition.py +15 -0
- omniload/src/testdata/fakebqcredentials.json +14 -0
- omniload/src/tiktok_ads/__init__.py +150 -0
- omniload/src/tiktok_ads/tiktok_helpers.py +130 -0
- omniload/src/time.py +11 -0
- omniload/src/trustpilot/__init__.py +48 -0
- omniload/src/trustpilot/client.py +48 -0
- omniload/src/version.py +6 -0
- omniload/src/wise/__init__.py +68 -0
- omniload/src/wise/client.py +63 -0
- omniload/src/zendesk/__init__.py +480 -0
- omniload/src/zendesk/helpers/__init__.py +39 -0
- omniload/src/zendesk/helpers/api_helpers.py +119 -0
- omniload/src/zendesk/helpers/credentials.py +68 -0
- omniload/src/zendesk/helpers/talk_api.py +132 -0
- omniload/src/zendesk/settings.py +71 -0
- omniload/src/zoom/__init__.py +99 -0
- omniload/src/zoom/helpers.py +102 -0
- omniload/testdata/.gitignore +2 -0
- omniload/testdata/create_replace.csv +21 -0
- omniload/testdata/delete_insert_expected.csv +6 -0
- omniload/testdata/delete_insert_part1.csv +5 -0
- omniload/testdata/delete_insert_part2.csv +6 -0
- omniload/testdata/merge_expected.csv +5 -0
- omniload/testdata/merge_part1.csv +4 -0
- omniload/testdata/merge_part2.csv +5 -0
- omniload/tests/unit/test_smartsheets.py +133 -0
- omniload-0.0.0.dev0.dist-info/METADATA +439 -0
- omniload-0.0.0.dev0.dist-info/RECORD +218 -0
- omniload-0.0.0.dev0.dist-info/WHEEL +4 -0
- omniload-0.0.0.dev0.dist-info/entry_points.txt +2 -0
- omniload-0.0.0.dev0.dist-info/licenses/LICENSE.Apache-2.0 +201 -0
- omniload-0.0.0.dev0.dist-info/licenses/LICENSE.md +21 -0
- omniload-0.0.0.dev0.dist-info/licenses/NOTICE +35 -0
|
@@ -0,0 +1,800 @@
|
|
|
1
|
+
# Copyright 2022-2025 ScaleVector
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
This is a module that provides a DLT source to retrieve data from multiple endpoints of the HubSpot API using a specified API key. The retrieved data is returned as a tuple of Dlt resources, one for each endpoint.
|
|
17
|
+
|
|
18
|
+
The source retrieves data from the following endpoints:
|
|
19
|
+
- CRM Companies
|
|
20
|
+
- CRM Contacts
|
|
21
|
+
- CRM Deals
|
|
22
|
+
- CRM Tickets
|
|
23
|
+
- CRM Products
|
|
24
|
+
- CRM Quotes
|
|
25
|
+
- Web Analytics Events
|
|
26
|
+
|
|
27
|
+
For each endpoint, a resource and transformer function are defined to retrieve data and transform it to a common format.
|
|
28
|
+
The resource functions yield the raw data retrieved from the API, while the transformer functions are used to retrieve
|
|
29
|
+
additional information from the Web Analytics Events endpoint.
|
|
30
|
+
|
|
31
|
+
The source also supports enabling Web Analytics Events for each endpoint by setting the corresponding enable flag to True.
|
|
32
|
+
|
|
33
|
+
Example:
|
|
34
|
+
To retrieve data from all endpoints, use the following code:
|
|
35
|
+
|
|
36
|
+
python
|
|
37
|
+
|
|
38
|
+
>>> resources = hubspot(api_key="your_api_key")
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence
|
|
42
|
+
from urllib.parse import parse_qs, quote, urlparse
|
|
43
|
+
|
|
44
|
+
import dlt
|
|
45
|
+
from dlt.common import pendulum
|
|
46
|
+
from dlt.common.typing import TDataItems
|
|
47
|
+
from dlt.sources import DltResource
|
|
48
|
+
|
|
49
|
+
from .helpers import (
|
|
50
|
+
_get_property_names,
|
|
51
|
+
fetch_data,
|
|
52
|
+
fetch_data_raw,
|
|
53
|
+
fetch_data_search,
|
|
54
|
+
fetch_property_history,
|
|
55
|
+
)
|
|
56
|
+
from .settings import (
|
|
57
|
+
ALL,
|
|
58
|
+
CRM_OBJECT_ENDPOINTS,
|
|
59
|
+
CRM_OWNERS_ENDPOINT,
|
|
60
|
+
CRM_SCHEMAS_ENDPOINT,
|
|
61
|
+
DEFAULT_CALL_PROPS,
|
|
62
|
+
DEFAULT_CART_PROPS,
|
|
63
|
+
DEFAULT_COMMERCE_PAYMENT_PROPS,
|
|
64
|
+
DEFAULT_COMPANY_PROPS,
|
|
65
|
+
DEFAULT_CONTACT_PROPS,
|
|
66
|
+
DEFAULT_DEAL_PROPS,
|
|
67
|
+
DEFAULT_DISCOUNT_PROPS,
|
|
68
|
+
DEFAULT_EMAIL_PROPS,
|
|
69
|
+
DEFAULT_FEE_PROPS,
|
|
70
|
+
DEFAULT_FEEDBACK_SUBMISSION_PROPS,
|
|
71
|
+
DEFAULT_INVOICE_PROPS,
|
|
72
|
+
DEFAULT_LINE_ITEM_PROPS,
|
|
73
|
+
DEFAULT_MEETING_PROPS,
|
|
74
|
+
DEFAULT_NOTE_PROPS,
|
|
75
|
+
DEFAULT_PRODUCT_PROPS,
|
|
76
|
+
DEFAULT_QUOTE_PROPS,
|
|
77
|
+
DEFAULT_TASK_PROPS,
|
|
78
|
+
DEFAULT_TAX_PROPS,
|
|
79
|
+
DEFAULT_TICKET_PROPS,
|
|
80
|
+
OBJECT_TYPE_PLURAL,
|
|
81
|
+
STARTDATE,
|
|
82
|
+
WEB_ANALYTICS_EVENTS_ENDPOINT,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
THubspotObjectType = Literal[
|
|
86
|
+
"company",
|
|
87
|
+
"contact",
|
|
88
|
+
"deal",
|
|
89
|
+
"ticket",
|
|
90
|
+
"product",
|
|
91
|
+
"quote",
|
|
92
|
+
"call",
|
|
93
|
+
"email",
|
|
94
|
+
"feedback_submission",
|
|
95
|
+
"line_item",
|
|
96
|
+
"meeting",
|
|
97
|
+
"note",
|
|
98
|
+
"task",
|
|
99
|
+
"cart",
|
|
100
|
+
"discount",
|
|
101
|
+
"fee",
|
|
102
|
+
"invoice",
|
|
103
|
+
"commerce_payment",
|
|
104
|
+
"tax",
|
|
105
|
+
]
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _last_value_to_ms(last_value) -> Optional[str]:
|
|
109
|
+
"""Convert dlt incremental last_value (ISO string or datetime) to ms timestamp string."""
|
|
110
|
+
if last_value is None:
|
|
111
|
+
return None
|
|
112
|
+
dt = (
|
|
113
|
+
pendulum.parse(last_value)
|
|
114
|
+
if isinstance(last_value, str)
|
|
115
|
+
else pendulum.instance(last_value)
|
|
116
|
+
)
|
|
117
|
+
return str(int(dt.timestamp() * 1000))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dlt.source(name="hubspot", max_table_nesting=0)
|
|
121
|
+
def hubspot(
|
|
122
|
+
api_key: str = dlt.secrets.value,
|
|
123
|
+
include_history: bool = False,
|
|
124
|
+
include_custom_props: bool = True,
|
|
125
|
+
custom_object: str = None,
|
|
126
|
+
start_date: Optional[Any] = None,
|
|
127
|
+
end_date: Optional[Any] = None,
|
|
128
|
+
) -> Sequence[DltResource]:
|
|
129
|
+
"""
|
|
130
|
+
A DLT source that retrieves data from the HubSpot API using the
|
|
131
|
+
specified API key.
|
|
132
|
+
|
|
133
|
+
This function retrieves data for several HubSpot API endpoints,
|
|
134
|
+
including companies, contacts, deals, tickets, products and web
|
|
135
|
+
analytics events. It returns a tuple of Dlt resources, one for
|
|
136
|
+
each endpoint.
|
|
137
|
+
|
|
138
|
+
Args:
|
|
139
|
+
api_key (Optional[str]):
|
|
140
|
+
The API key used to authenticate with the HubSpot API. Defaults
|
|
141
|
+
to dlt.secrets.value.
|
|
142
|
+
include_history (Optional[bool]):
|
|
143
|
+
Whether to load history of property changes along with entities.
|
|
144
|
+
The history entries are loaded to separate tables.
|
|
145
|
+
|
|
146
|
+
Returns:
|
|
147
|
+
Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint.
|
|
148
|
+
|
|
149
|
+
Notes:
|
|
150
|
+
This function uses the `fetch_data` function to retrieve data from the
|
|
151
|
+
HubSpot CRM API. The API key is passed to `fetch_data` as the
|
|
152
|
+
`api_key` argument.
|
|
153
|
+
"""
|
|
154
|
+
|
|
155
|
+
if start_date is None:
|
|
156
|
+
start_date = "1970-01-01T00:00:00Z"
|
|
157
|
+
elif not isinstance(start_date, str):
|
|
158
|
+
start_date = pendulum.instance(start_date).to_iso8601_string()
|
|
159
|
+
if end_date is not None and not isinstance(end_date, str):
|
|
160
|
+
end_date = pendulum.instance(end_date).to_iso8601_string()
|
|
161
|
+
|
|
162
|
+
@dlt.resource(
|
|
163
|
+
name="companies", write_disposition="merge", primary_key=["hs_object_id"]
|
|
164
|
+
)
|
|
165
|
+
def companies(
|
|
166
|
+
api_key: str = api_key,
|
|
167
|
+
include_history: bool = include_history,
|
|
168
|
+
include_custom_props: bool = include_custom_props,
|
|
169
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
170
|
+
"hs_lastmodifieddate",
|
|
171
|
+
initial_value=start_date,
|
|
172
|
+
end_value=end_date,
|
|
173
|
+
),
|
|
174
|
+
) -> Iterator[TDataItems]:
|
|
175
|
+
"""Hubspot companies resource"""
|
|
176
|
+
yield from crm_objects(
|
|
177
|
+
"company",
|
|
178
|
+
api_key,
|
|
179
|
+
include_history=include_history,
|
|
180
|
+
props=DEFAULT_COMPANY_PROPS,
|
|
181
|
+
include_custom_props=include_custom_props,
|
|
182
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
183
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
@dlt.resource(
|
|
187
|
+
name="contacts", write_disposition="merge", primary_key=["hs_object_id"]
|
|
188
|
+
)
|
|
189
|
+
def contacts(
|
|
190
|
+
api_key: str = api_key,
|
|
191
|
+
include_history: bool = include_history,
|
|
192
|
+
include_custom_props: bool = include_custom_props,
|
|
193
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
194
|
+
"lastmodifieddate",
|
|
195
|
+
initial_value=start_date,
|
|
196
|
+
end_value=end_date,
|
|
197
|
+
),
|
|
198
|
+
) -> Iterator[TDataItems]:
|
|
199
|
+
"""Hubspot contacts resource"""
|
|
200
|
+
yield from crm_objects(
|
|
201
|
+
"contact",
|
|
202
|
+
api_key,
|
|
203
|
+
include_history,
|
|
204
|
+
DEFAULT_CONTACT_PROPS,
|
|
205
|
+
include_custom_props,
|
|
206
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
207
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
@dlt.resource(name="deals", write_disposition="merge", primary_key=["hs_object_id"])
|
|
211
|
+
def deals(
|
|
212
|
+
api_key: str = api_key,
|
|
213
|
+
include_history: bool = include_history,
|
|
214
|
+
include_custom_props: bool = include_custom_props,
|
|
215
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
216
|
+
"hs_lastmodifieddate",
|
|
217
|
+
initial_value=start_date,
|
|
218
|
+
end_value=end_date,
|
|
219
|
+
),
|
|
220
|
+
) -> Iterator[TDataItems]:
|
|
221
|
+
"""Hubspot deals resource"""
|
|
222
|
+
yield from crm_objects(
|
|
223
|
+
"deal",
|
|
224
|
+
api_key,
|
|
225
|
+
include_history,
|
|
226
|
+
DEFAULT_DEAL_PROPS,
|
|
227
|
+
include_custom_props,
|
|
228
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
229
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
@dlt.resource(
|
|
233
|
+
name="tickets", write_disposition="merge", primary_key=["hs_object_id"]
|
|
234
|
+
)
|
|
235
|
+
def tickets(
|
|
236
|
+
api_key: str = api_key,
|
|
237
|
+
include_history: bool = include_history,
|
|
238
|
+
include_custom_props: bool = include_custom_props,
|
|
239
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
240
|
+
"hs_lastmodifieddate",
|
|
241
|
+
initial_value=start_date,
|
|
242
|
+
end_value=end_date,
|
|
243
|
+
),
|
|
244
|
+
) -> Iterator[TDataItems]:
|
|
245
|
+
"""Hubspot tickets resource"""
|
|
246
|
+
yield from crm_objects(
|
|
247
|
+
"ticket",
|
|
248
|
+
api_key,
|
|
249
|
+
include_history,
|
|
250
|
+
DEFAULT_TICKET_PROPS,
|
|
251
|
+
include_custom_props,
|
|
252
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
253
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
@dlt.resource(
|
|
257
|
+
name="products", write_disposition="merge", primary_key=["hs_object_id"]
|
|
258
|
+
)
|
|
259
|
+
def products(
|
|
260
|
+
api_key: str = api_key,
|
|
261
|
+
include_history: bool = include_history,
|
|
262
|
+
include_custom_props: bool = include_custom_props,
|
|
263
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
264
|
+
"hs_lastmodifieddate",
|
|
265
|
+
initial_value=start_date,
|
|
266
|
+
end_value=end_date,
|
|
267
|
+
),
|
|
268
|
+
) -> Iterator[TDataItems]:
|
|
269
|
+
"""Hubspot products resource"""
|
|
270
|
+
yield from crm_objects(
|
|
271
|
+
"product",
|
|
272
|
+
api_key,
|
|
273
|
+
include_history,
|
|
274
|
+
DEFAULT_PRODUCT_PROPS,
|
|
275
|
+
include_custom_props,
|
|
276
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
277
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
@dlt.resource(name="calls", write_disposition="merge", primary_key=["hs_object_id"])
|
|
281
|
+
def calls(
|
|
282
|
+
api_key: str = api_key,
|
|
283
|
+
include_history: bool = include_history,
|
|
284
|
+
include_custom_props: bool = include_custom_props,
|
|
285
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
286
|
+
"hs_lastmodifieddate",
|
|
287
|
+
initial_value=start_date,
|
|
288
|
+
end_value=end_date,
|
|
289
|
+
),
|
|
290
|
+
) -> Iterator[TDataItems]:
|
|
291
|
+
"""Hubspot calls resource"""
|
|
292
|
+
yield from crm_objects(
|
|
293
|
+
"call",
|
|
294
|
+
api_key,
|
|
295
|
+
include_history,
|
|
296
|
+
DEFAULT_CALL_PROPS,
|
|
297
|
+
include_custom_props,
|
|
298
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
299
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
@dlt.resource(
|
|
303
|
+
name="emails", write_disposition="merge", primary_key=["hs_object_id"]
|
|
304
|
+
)
|
|
305
|
+
def emails(
|
|
306
|
+
api_key: str = api_key,
|
|
307
|
+
include_history: bool = include_history,
|
|
308
|
+
include_custom_props: bool = include_custom_props,
|
|
309
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
310
|
+
"hs_lastmodifieddate",
|
|
311
|
+
initial_value=start_date,
|
|
312
|
+
end_value=end_date,
|
|
313
|
+
),
|
|
314
|
+
) -> Iterator[TDataItems]:
|
|
315
|
+
"""Hubspot emails resource"""
|
|
316
|
+
yield from crm_objects(
|
|
317
|
+
"email",
|
|
318
|
+
api_key,
|
|
319
|
+
include_history,
|
|
320
|
+
DEFAULT_EMAIL_PROPS,
|
|
321
|
+
include_custom_props,
|
|
322
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
323
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
@dlt.resource(
|
|
327
|
+
name="feedback_submissions",
|
|
328
|
+
write_disposition="merge",
|
|
329
|
+
primary_key=["hs_object_id"],
|
|
330
|
+
)
|
|
331
|
+
def feedback_submissions(
|
|
332
|
+
api_key: str = api_key,
|
|
333
|
+
include_history: bool = include_history,
|
|
334
|
+
include_custom_props: bool = include_custom_props,
|
|
335
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
336
|
+
"hs_lastmodifieddate",
|
|
337
|
+
initial_value=start_date,
|
|
338
|
+
end_value=end_date,
|
|
339
|
+
),
|
|
340
|
+
) -> Iterator[TDataItems]:
|
|
341
|
+
"""Hubspot feedback submissions resource"""
|
|
342
|
+
yield from crm_objects(
|
|
343
|
+
"feedback_submission",
|
|
344
|
+
api_key,
|
|
345
|
+
include_history,
|
|
346
|
+
DEFAULT_FEEDBACK_SUBMISSION_PROPS,
|
|
347
|
+
include_custom_props,
|
|
348
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
349
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
350
|
+
)
|
|
351
|
+
|
|
352
|
+
@dlt.resource(
|
|
353
|
+
name="line_items", write_disposition="merge", primary_key=["hs_object_id"]
|
|
354
|
+
)
|
|
355
|
+
def line_items(
|
|
356
|
+
api_key: str = api_key,
|
|
357
|
+
include_history: bool = include_history,
|
|
358
|
+
include_custom_props: bool = include_custom_props,
|
|
359
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
360
|
+
"hs_lastmodifieddate",
|
|
361
|
+
initial_value=start_date,
|
|
362
|
+
end_value=end_date,
|
|
363
|
+
),
|
|
364
|
+
) -> Iterator[TDataItems]:
|
|
365
|
+
"""Hubspot line items resource"""
|
|
366
|
+
yield from crm_objects(
|
|
367
|
+
"line_item",
|
|
368
|
+
api_key,
|
|
369
|
+
include_history,
|
|
370
|
+
DEFAULT_LINE_ITEM_PROPS,
|
|
371
|
+
include_custom_props,
|
|
372
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
373
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
@dlt.resource(
|
|
377
|
+
name="meetings", write_disposition="merge", primary_key=["hs_object_id"]
|
|
378
|
+
)
|
|
379
|
+
def meetings(
|
|
380
|
+
api_key: str = api_key,
|
|
381
|
+
include_history: bool = include_history,
|
|
382
|
+
include_custom_props: bool = include_custom_props,
|
|
383
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
384
|
+
"hs_lastmodifieddate",
|
|
385
|
+
initial_value=start_date,
|
|
386
|
+
end_value=end_date,
|
|
387
|
+
),
|
|
388
|
+
) -> Iterator[TDataItems]:
|
|
389
|
+
"""Hubspot meetings resource"""
|
|
390
|
+
yield from crm_objects(
|
|
391
|
+
"meeting",
|
|
392
|
+
api_key,
|
|
393
|
+
include_history,
|
|
394
|
+
DEFAULT_MEETING_PROPS,
|
|
395
|
+
include_custom_props,
|
|
396
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
397
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
@dlt.resource(name="notes", write_disposition="merge", primary_key=["hs_object_id"])
|
|
401
|
+
def notes(
|
|
402
|
+
api_key: str = api_key,
|
|
403
|
+
include_history: bool = include_history,
|
|
404
|
+
include_custom_props: bool = include_custom_props,
|
|
405
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
406
|
+
"hs_lastmodifieddate",
|
|
407
|
+
initial_value=start_date,
|
|
408
|
+
end_value=end_date,
|
|
409
|
+
),
|
|
410
|
+
) -> Iterator[TDataItems]:
|
|
411
|
+
"""Hubspot notes resource"""
|
|
412
|
+
yield from crm_objects(
|
|
413
|
+
"note",
|
|
414
|
+
api_key,
|
|
415
|
+
include_history,
|
|
416
|
+
DEFAULT_NOTE_PROPS,
|
|
417
|
+
include_custom_props,
|
|
418
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
419
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
@dlt.resource(name="tasks", write_disposition="merge", primary_key=["hs_object_id"])
|
|
423
|
+
def tasks(
|
|
424
|
+
api_key: str = api_key,
|
|
425
|
+
include_history: bool = include_history,
|
|
426
|
+
include_custom_props: bool = include_custom_props,
|
|
427
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
428
|
+
"hs_lastmodifieddate",
|
|
429
|
+
initial_value=start_date,
|
|
430
|
+
end_value=end_date,
|
|
431
|
+
),
|
|
432
|
+
) -> Iterator[TDataItems]:
|
|
433
|
+
"""Hubspot tasks resource"""
|
|
434
|
+
yield from crm_objects(
|
|
435
|
+
"task",
|
|
436
|
+
api_key,
|
|
437
|
+
include_history,
|
|
438
|
+
DEFAULT_TASK_PROPS,
|
|
439
|
+
include_custom_props,
|
|
440
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
441
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
442
|
+
)
|
|
443
|
+
|
|
444
|
+
@dlt.resource(name="carts", write_disposition="merge", primary_key=["hs_object_id"])
|
|
445
|
+
def carts(
|
|
446
|
+
api_key: str = api_key,
|
|
447
|
+
include_history: bool = include_history,
|
|
448
|
+
include_custom_props: bool = include_custom_props,
|
|
449
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
450
|
+
"hs_lastmodifieddate",
|
|
451
|
+
initial_value=start_date,
|
|
452
|
+
end_value=end_date,
|
|
453
|
+
),
|
|
454
|
+
) -> Iterator[TDataItems]:
|
|
455
|
+
"""Hubspot carts resource"""
|
|
456
|
+
yield from crm_objects(
|
|
457
|
+
"cart",
|
|
458
|
+
api_key,
|
|
459
|
+
include_history,
|
|
460
|
+
DEFAULT_CART_PROPS,
|
|
461
|
+
include_custom_props,
|
|
462
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
463
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
@dlt.resource(
|
|
467
|
+
name="discounts", write_disposition="merge", primary_key=["hs_object_id"]
|
|
468
|
+
)
|
|
469
|
+
def discounts(
|
|
470
|
+
api_key: str = api_key,
|
|
471
|
+
include_history: bool = include_history,
|
|
472
|
+
include_custom_props: bool = include_custom_props,
|
|
473
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
474
|
+
"hs_lastmodifieddate",
|
|
475
|
+
initial_value=start_date,
|
|
476
|
+
end_value=end_date,
|
|
477
|
+
),
|
|
478
|
+
) -> Iterator[TDataItems]:
|
|
479
|
+
"""Hubspot discounts resource"""
|
|
480
|
+
yield from crm_objects(
|
|
481
|
+
"discount",
|
|
482
|
+
api_key,
|
|
483
|
+
include_history,
|
|
484
|
+
DEFAULT_DISCOUNT_PROPS,
|
|
485
|
+
include_custom_props,
|
|
486
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
487
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
488
|
+
)
|
|
489
|
+
|
|
490
|
+
@dlt.resource(name="fees", write_disposition="merge", primary_key=["hs_object_id"])
|
|
491
|
+
def fees(
|
|
492
|
+
api_key: str = api_key,
|
|
493
|
+
include_history: bool = include_history,
|
|
494
|
+
include_custom_props: bool = include_custom_props,
|
|
495
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
496
|
+
"hs_lastmodifieddate",
|
|
497
|
+
initial_value=start_date,
|
|
498
|
+
end_value=end_date,
|
|
499
|
+
),
|
|
500
|
+
) -> Iterator[TDataItems]:
|
|
501
|
+
"""Hubspot fees resource"""
|
|
502
|
+
yield from crm_objects(
|
|
503
|
+
"fee",
|
|
504
|
+
api_key,
|
|
505
|
+
include_history,
|
|
506
|
+
DEFAULT_FEE_PROPS,
|
|
507
|
+
include_custom_props,
|
|
508
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
509
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
510
|
+
)
|
|
511
|
+
|
|
512
|
+
@dlt.resource(
|
|
513
|
+
name="invoices", write_disposition="merge", primary_key=["hs_object_id"]
|
|
514
|
+
)
|
|
515
|
+
def invoices(
|
|
516
|
+
api_key: str = api_key,
|
|
517
|
+
include_history: bool = include_history,
|
|
518
|
+
include_custom_props: bool = include_custom_props,
|
|
519
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
520
|
+
"hs_lastmodifieddate",
|
|
521
|
+
initial_value=start_date,
|
|
522
|
+
end_value=end_date,
|
|
523
|
+
),
|
|
524
|
+
) -> Iterator[TDataItems]:
|
|
525
|
+
"""Hubspot invoices resource"""
|
|
526
|
+
yield from crm_objects(
|
|
527
|
+
"invoice",
|
|
528
|
+
api_key,
|
|
529
|
+
include_history,
|
|
530
|
+
DEFAULT_INVOICE_PROPS,
|
|
531
|
+
include_custom_props,
|
|
532
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
533
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
@dlt.resource(
|
|
537
|
+
name="commerce_payments",
|
|
538
|
+
write_disposition="merge",
|
|
539
|
+
primary_key=["hs_object_id"],
|
|
540
|
+
)
|
|
541
|
+
def commerce_payments(
|
|
542
|
+
api_key: str = api_key,
|
|
543
|
+
include_history: bool = include_history,
|
|
544
|
+
include_custom_props: bool = include_custom_props,
|
|
545
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
546
|
+
"hs_lastmodifieddate",
|
|
547
|
+
initial_value=start_date,
|
|
548
|
+
end_value=end_date,
|
|
549
|
+
),
|
|
550
|
+
) -> Iterator[TDataItems]:
|
|
551
|
+
"""Hubspot commerce payments resource"""
|
|
552
|
+
yield from crm_objects(
|
|
553
|
+
"commerce_payment",
|
|
554
|
+
api_key,
|
|
555
|
+
include_history,
|
|
556
|
+
DEFAULT_COMMERCE_PAYMENT_PROPS,
|
|
557
|
+
include_custom_props,
|
|
558
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
559
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
@dlt.resource(name="taxes", write_disposition="merge", primary_key=["hs_object_id"])
|
|
563
|
+
def taxes(
|
|
564
|
+
api_key: str = api_key,
|
|
565
|
+
include_history: bool = include_history,
|
|
566
|
+
include_custom_props: bool = include_custom_props,
|
|
567
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
568
|
+
"hs_lastmodifieddate",
|
|
569
|
+
initial_value=start_date,
|
|
570
|
+
end_value=end_date,
|
|
571
|
+
),
|
|
572
|
+
) -> Iterator[TDataItems]:
|
|
573
|
+
"""Hubspot taxes resource"""
|
|
574
|
+
yield from crm_objects(
|
|
575
|
+
"tax",
|
|
576
|
+
api_key,
|
|
577
|
+
include_history,
|
|
578
|
+
DEFAULT_TAX_PROPS,
|
|
579
|
+
include_custom_props,
|
|
580
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
581
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
@dlt.resource(
|
|
585
|
+
name="quotes", write_disposition="merge", primary_key=["hs_object_id"]
|
|
586
|
+
)
|
|
587
|
+
def quotes(
|
|
588
|
+
api_key: str = api_key,
|
|
589
|
+
include_history: bool = include_history,
|
|
590
|
+
include_custom_props: bool = include_custom_props,
|
|
591
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
592
|
+
"hs_lastmodifieddate",
|
|
593
|
+
initial_value=start_date,
|
|
594
|
+
end_value=end_date,
|
|
595
|
+
),
|
|
596
|
+
) -> Iterator[TDataItems]:
|
|
597
|
+
"""Hubspot quotes resource"""
|
|
598
|
+
yield from crm_objects(
|
|
599
|
+
"quote",
|
|
600
|
+
api_key,
|
|
601
|
+
include_history,
|
|
602
|
+
DEFAULT_QUOTE_PROPS,
|
|
603
|
+
include_custom_props,
|
|
604
|
+
start_date_ms=_last_value_to_ms(updated_at.last_value),
|
|
605
|
+
end_date_ms=_last_value_to_ms(end_date),
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
@dlt.resource(name="owners", write_disposition="merge", primary_key="id")
|
|
609
|
+
def owners(
|
|
610
|
+
api_key: str = api_key,
|
|
611
|
+
) -> Iterator[TDataItems]:
|
|
612
|
+
"""Hubspot owners resource"""
|
|
613
|
+
yield from fetch_data(CRM_OWNERS_ENDPOINT, api_key, resource_name="owners")
|
|
614
|
+
|
|
615
|
+
@dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
|
|
616
|
+
def schemas(
|
|
617
|
+
api_key: str = api_key,
|
|
618
|
+
) -> Iterator[TDataItems]:
|
|
619
|
+
"""Hubspot schemas resource"""
|
|
620
|
+
yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key, resource_name="schemas")
|
|
621
|
+
|
|
622
|
+
@dlt.resource(write_disposition="merge", primary_key="hs_object_id")
|
|
623
|
+
def custom(
|
|
624
|
+
api_key: str = api_key,
|
|
625
|
+
custom_object_name: str = custom_object,
|
|
626
|
+
) -> Iterator[TDataItems]:
|
|
627
|
+
custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
|
|
628
|
+
object_type_id = None
|
|
629
|
+
associations = None
|
|
630
|
+
if ":" in custom_object_name:
|
|
631
|
+
fields = custom_object_name.split(":")
|
|
632
|
+
if len(fields) == 2:
|
|
633
|
+
custom_object_name = fields[0]
|
|
634
|
+
associations = fields[1]
|
|
635
|
+
|
|
636
|
+
custom_object_lowercase = custom_object_name.lower()
|
|
637
|
+
|
|
638
|
+
for custom_object in custom_objects["results"]:
|
|
639
|
+
if custom_object["name"].lower() == custom_object_lowercase:
|
|
640
|
+
object_type_id = custom_object["objectTypeId"]
|
|
641
|
+
break
|
|
642
|
+
|
|
643
|
+
# sometimes people use the plural name of the object type by accident, we should try to match that if we can
|
|
644
|
+
if "labels" in custom_object:
|
|
645
|
+
if custom_object_lowercase == custom_object["labels"]["plural"].lower():
|
|
646
|
+
object_type_id = custom_object["objectTypeId"]
|
|
647
|
+
break
|
|
648
|
+
|
|
649
|
+
if object_type_id is None:
|
|
650
|
+
raise ValueError(f"There is no such custom object as {custom_object_name}")
|
|
651
|
+
custom_object_properties = f"crm/v3/properties/{object_type_id}"
|
|
652
|
+
|
|
653
|
+
props_pages = fetch_data(custom_object_properties, api_key)
|
|
654
|
+
props = []
|
|
655
|
+
for page in props_pages:
|
|
656
|
+
props.extend([prop["name"] for prop in page])
|
|
657
|
+
props = ",".join(sorted(list(set(props))))
|
|
658
|
+
|
|
659
|
+
custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
|
|
660
|
+
if associations:
|
|
661
|
+
custom_object_endpoint += f"&associations={associations}"
|
|
662
|
+
|
|
663
|
+
"""Hubspot custom object details resource"""
|
|
664
|
+
yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")
|
|
665
|
+
|
|
666
|
+
return (
|
|
667
|
+
companies,
|
|
668
|
+
contacts,
|
|
669
|
+
deals,
|
|
670
|
+
tickets,
|
|
671
|
+
products,
|
|
672
|
+
quotes,
|
|
673
|
+
calls,
|
|
674
|
+
emails,
|
|
675
|
+
feedback_submissions,
|
|
676
|
+
line_items,
|
|
677
|
+
meetings,
|
|
678
|
+
notes,
|
|
679
|
+
tasks,
|
|
680
|
+
carts,
|
|
681
|
+
discounts,
|
|
682
|
+
fees,
|
|
683
|
+
invoices,
|
|
684
|
+
commerce_payments,
|
|
685
|
+
taxes,
|
|
686
|
+
owners,
|
|
687
|
+
schemas,
|
|
688
|
+
custom,
|
|
689
|
+
)
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def crm_objects(
|
|
693
|
+
object_type: str,
|
|
694
|
+
api_key: str = dlt.secrets.value,
|
|
695
|
+
include_history: bool = False,
|
|
696
|
+
props: Sequence[str] = None,
|
|
697
|
+
include_custom_props: bool = True,
|
|
698
|
+
start_date_ms: Optional[str] = None,
|
|
699
|
+
end_date_ms: Optional[str] = None,
|
|
700
|
+
) -> Iterator[TDataItems]:
|
|
701
|
+
"""Building blocks for CRM resources."""
|
|
702
|
+
if props == ALL:
|
|
703
|
+
props = list(_get_property_names(api_key, object_type))
|
|
704
|
+
|
|
705
|
+
if include_custom_props:
|
|
706
|
+
props += _get_property_names(api_key, object_type)
|
|
707
|
+
|
|
708
|
+
props = ",".join(sorted(list(set(props))))
|
|
709
|
+
|
|
710
|
+
if start_date_ms is not None:
|
|
711
|
+
_qs = parse_qs(urlparse(CRM_OBJECT_ENDPOINTS[object_type]).query)
|
|
712
|
+
assoc_types = [t for t in _qs.get("associations", [""])[0].split(",") if t]
|
|
713
|
+
yield from fetch_data_search(
|
|
714
|
+
object_type,
|
|
715
|
+
api_key,
|
|
716
|
+
props,
|
|
717
|
+
start_date_ms,
|
|
718
|
+
end_date_ms=end_date_ms,
|
|
719
|
+
association_types=assoc_types,
|
|
720
|
+
)
|
|
721
|
+
return
|
|
722
|
+
|
|
723
|
+
params = {"properties": props, "limit": 100}
|
|
724
|
+
|
|
725
|
+
yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, params=params)
|
|
726
|
+
if include_history:
|
|
727
|
+
# Get history separately, as requesting both all properties and history together
|
|
728
|
+
# is likely to hit hubspot's URL length limit
|
|
729
|
+
for history_entries in fetch_property_history(
|
|
730
|
+
CRM_OBJECT_ENDPOINTS[object_type],
|
|
731
|
+
api_key,
|
|
732
|
+
props,
|
|
733
|
+
):
|
|
734
|
+
yield dlt.mark.with_table_name(
|
|
735
|
+
history_entries,
|
|
736
|
+
OBJECT_TYPE_PLURAL[object_type] + "_property_history",
|
|
737
|
+
)
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
@dlt.resource
|
|
741
|
+
def hubspot_events_for_objects(
|
|
742
|
+
object_type: THubspotObjectType,
|
|
743
|
+
object_ids: List[str],
|
|
744
|
+
api_key: str = dlt.secrets.value,
|
|
745
|
+
start_date: pendulum.DateTime = STARTDATE,
|
|
746
|
+
) -> DltResource:
|
|
747
|
+
"""
|
|
748
|
+
A standalone DLT resources that retrieves web analytics events from the HubSpot API for a particular object type and list of object ids.
|
|
749
|
+
|
|
750
|
+
Args:
|
|
751
|
+
object_type(THubspotObjectType, required): One of the hubspot object types see definition of THubspotObjectType literal
|
|
752
|
+
object_ids: (List[THubspotObjectType], required): List of object ids to track events
|
|
753
|
+
api_key (str, optional): The API key used to authenticate with the HubSpot API. Defaults to dlt.secrets.value.
|
|
754
|
+
start_date (datetime, optional): The initial date time from which start getting events, default to STARTDATE
|
|
755
|
+
|
|
756
|
+
Returns:
|
|
757
|
+
incremental dlt resource to track events for objects from the list
|
|
758
|
+
"""
|
|
759
|
+
|
|
760
|
+
end_date = pendulum.now().isoformat()
|
|
761
|
+
name = object_type + "_events"
|
|
762
|
+
|
|
763
|
+
def get_web_analytics_events(
|
|
764
|
+
occurred_at: dlt.sources.incremental[str],
|
|
765
|
+
) -> Iterator[List[Dict[str, Any]]]:
|
|
766
|
+
"""
|
|
767
|
+
A helper function that retrieves web analytics events for a given object type from the HubSpot API.
|
|
768
|
+
|
|
769
|
+
Args:
|
|
770
|
+
object_type (str): The type of object for which to retrieve web analytics events.
|
|
771
|
+
|
|
772
|
+
Yields:
|
|
773
|
+
dict: A dictionary representing a web analytics event.
|
|
774
|
+
"""
|
|
775
|
+
for object_id in object_ids:
|
|
776
|
+
yield from fetch_data(
|
|
777
|
+
WEB_ANALYTICS_EVENTS_ENDPOINT.format(
|
|
778
|
+
objectType=object_type,
|
|
779
|
+
objectId=object_id,
|
|
780
|
+
occurredAfter=quote(occurred_at.last_value),
|
|
781
|
+
occurredBefore=quote(end_date),
|
|
782
|
+
),
|
|
783
|
+
api_key=api_key,
|
|
784
|
+
)
|
|
785
|
+
|
|
786
|
+
return dlt.resource(
|
|
787
|
+
get_web_analytics_events,
|
|
788
|
+
name=name,
|
|
789
|
+
primary_key="id",
|
|
790
|
+
write_disposition="append",
|
|
791
|
+
selected=True,
|
|
792
|
+
table_name=lambda e: name + "_" + str(e["eventType"]),
|
|
793
|
+
)(
|
|
794
|
+
dlt.sources.incremental(
|
|
795
|
+
"occurredAt",
|
|
796
|
+
initial_value=start_date.isoformat(),
|
|
797
|
+
range_end="closed",
|
|
798
|
+
range_start="closed",
|
|
799
|
+
)
|
|
800
|
+
)
|