omniload 0.0.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. omniload/conftest.py +72 -0
  2. omniload/main.py +810 -0
  3. omniload/src/.gitignore +10 -0
  4. omniload/src/adjust/__init__.py +108 -0
  5. omniload/src/adjust/adjust_helpers.py +122 -0
  6. omniload/src/airtable/__init__.py +84 -0
  7. omniload/src/allium/__init__.py +128 -0
  8. omniload/src/anthropic/__init__.py +277 -0
  9. omniload/src/anthropic/helpers.py +525 -0
  10. omniload/src/applovin/__init__.py +316 -0
  11. omniload/src/applovin_max/__init__.py +117 -0
  12. omniload/src/appsflyer/__init__.py +325 -0
  13. omniload/src/appsflyer/client.py +110 -0
  14. omniload/src/appstore/__init__.py +142 -0
  15. omniload/src/appstore/client.py +126 -0
  16. omniload/src/appstore/errors.py +15 -0
  17. omniload/src/appstore/models.py +117 -0
  18. omniload/src/appstore/resources.py +179 -0
  19. omniload/src/arrow/__init__.py +81 -0
  20. omniload/src/asana_source/__init__.py +281 -0
  21. omniload/src/asana_source/helpers.py +30 -0
  22. omniload/src/asana_source/settings.py +158 -0
  23. omniload/src/attio/__init__.py +102 -0
  24. omniload/src/attio/helpers.py +65 -0
  25. omniload/src/blob.py +95 -0
  26. omniload/src/bruin/__init__.py +76 -0
  27. omniload/src/chess/__init__.py +180 -0
  28. omniload/src/chess/helpers.py +35 -0
  29. omniload/src/chess/settings.py +18 -0
  30. omniload/src/clickup/__init__.py +85 -0
  31. omniload/src/clickup/helpers.py +47 -0
  32. omniload/src/collector/spinner.py +43 -0
  33. omniload/src/couchbase_source/__init__.py +118 -0
  34. omniload/src/couchbase_source/helpers.py +135 -0
  35. omniload/src/cursor/__init__.py +83 -0
  36. omniload/src/cursor/helpers.py +188 -0
  37. omniload/src/customer_io/__init__.py +486 -0
  38. omniload/src/customer_io/helpers.py +530 -0
  39. omniload/src/destinations.py +982 -0
  40. omniload/src/docebo/__init__.py +589 -0
  41. omniload/src/docebo/client.py +435 -0
  42. omniload/src/docebo/helpers.py +97 -0
  43. omniload/src/dune/__init__.py +104 -0
  44. omniload/src/dune/helpers.py +108 -0
  45. omniload/src/dynamodb/__init__.py +86 -0
  46. omniload/src/elasticsearch/__init__.py +80 -0
  47. omniload/src/elasticsearch/helpers.py +141 -0
  48. omniload/src/errors.py +26 -0
  49. omniload/src/facebook_ads/__init__.py +403 -0
  50. omniload/src/facebook_ads/exceptions.py +19 -0
  51. omniload/src/facebook_ads/helpers.py +296 -0
  52. omniload/src/facebook_ads/settings.py +224 -0
  53. omniload/src/facebook_ads/utils.py +53 -0
  54. omniload/src/factory.py +305 -0
  55. omniload/src/filesystem/__init__.py +133 -0
  56. omniload/src/filesystem/helpers.py +114 -0
  57. omniload/src/filesystem/readers.py +187 -0
  58. omniload/src/filters.py +62 -0
  59. omniload/src/fireflies/__init__.py +151 -0
  60. omniload/src/fireflies/helpers.py +753 -0
  61. omniload/src/fluxx/__init__.py +10013 -0
  62. omniload/src/fluxx/helpers.py +233 -0
  63. omniload/src/frankfurter/__init__.py +157 -0
  64. omniload/src/frankfurter/helpers.py +48 -0
  65. omniload/src/freshdesk/__init__.py +103 -0
  66. omniload/src/freshdesk/freshdesk_client.py +151 -0
  67. omniload/src/freshdesk/settings.py +23 -0
  68. omniload/src/fundraiseup/__init__.py +95 -0
  69. omniload/src/fundraiseup/client.py +81 -0
  70. omniload/src/github/__init__.py +202 -0
  71. omniload/src/github/helpers.py +207 -0
  72. omniload/src/github/queries.py +129 -0
  73. omniload/src/github/settings.py +24 -0
  74. omniload/src/google_ads/__init__.py +198 -0
  75. omniload/src/google_ads/field.py +17 -0
  76. omniload/src/google_ads/metrics.py +254 -0
  77. omniload/src/google_ads/predicates.py +37 -0
  78. omniload/src/google_ads/reports.py +411 -0
  79. omniload/src/google_ads/test_google_ads.py +184 -0
  80. omniload/src/google_analytics/__init__.py +144 -0
  81. omniload/src/google_analytics/helpers.py +312 -0
  82. omniload/src/google_sheets/README.md +95 -0
  83. omniload/src/google_sheets/__init__.py +166 -0
  84. omniload/src/google_sheets/helpers/__init__.py +15 -0
  85. omniload/src/google_sheets/helpers/api_calls.py +160 -0
  86. omniload/src/google_sheets/helpers/data_processing.py +316 -0
  87. omniload/src/gorgias/__init__.py +595 -0
  88. omniload/src/gorgias/helpers.py +166 -0
  89. omniload/src/hostaway/__init__.py +302 -0
  90. omniload/src/hostaway/client.py +288 -0
  91. omniload/src/http/__init__.py +38 -0
  92. omniload/src/http/readers.py +146 -0
  93. omniload/src/http_client.py +24 -0
  94. omniload/src/hubspot/__init__.py +800 -0
  95. omniload/src/hubspot/helpers.py +417 -0
  96. omniload/src/hubspot/settings.py +329 -0
  97. omniload/src/indeed/__init__.py +153 -0
  98. omniload/src/indeed/helpers.py +228 -0
  99. omniload/src/influxdb/__init__.py +46 -0
  100. omniload/src/influxdb/client.py +34 -0
  101. omniload/src/intercom/__init__.py +142 -0
  102. omniload/src/intercom/helpers.py +674 -0
  103. omniload/src/intercom/settings.py +279 -0
  104. omniload/src/isoc_pulse/__init__.py +159 -0
  105. omniload/src/jira_source/__init__.py +377 -0
  106. omniload/src/jira_source/helpers.py +510 -0
  107. omniload/src/jira_source/settings.py +184 -0
  108. omniload/src/kafka/__init__.py +120 -0
  109. omniload/src/kafka/helpers.py +241 -0
  110. omniload/src/kinesis/__init__.py +153 -0
  111. omniload/src/kinesis/helpers.py +96 -0
  112. omniload/src/klaviyo/__init__.py +237 -0
  113. omniload/src/klaviyo/client.py +212 -0
  114. omniload/src/klaviyo/helpers.py +19 -0
  115. omniload/src/linear/__init__.py +634 -0
  116. omniload/src/linear/helpers.py +111 -0
  117. omniload/src/linkedin_ads/__init__.py +266 -0
  118. omniload/src/linkedin_ads/dimension_time_enum.py +17 -0
  119. omniload/src/linkedin_ads/helpers.py +246 -0
  120. omniload/src/loader.py +69 -0
  121. omniload/src/mailchimp/__init__.py +126 -0
  122. omniload/src/mailchimp/helpers.py +226 -0
  123. omniload/src/mailchimp/settings.py +164 -0
  124. omniload/src/masking.py +344 -0
  125. omniload/src/mixpanel/__init__.py +62 -0
  126. omniload/src/mixpanel/client.py +104 -0
  127. omniload/src/monday/__init__.py +246 -0
  128. omniload/src/monday/helpers.py +392 -0
  129. omniload/src/monday/settings.py +325 -0
  130. omniload/src/mongodb/__init__.py +281 -0
  131. omniload/src/mongodb/helpers.py +975 -0
  132. omniload/src/notion/__init__.py +69 -0
  133. omniload/src/notion/helpers/__init__.py +14 -0
  134. omniload/src/notion/helpers/client.py +178 -0
  135. omniload/src/notion/helpers/database.py +92 -0
  136. omniload/src/notion/settings.py +17 -0
  137. omniload/src/partition.py +32 -0
  138. omniload/src/personio/__init__.py +345 -0
  139. omniload/src/personio/helpers.py +100 -0
  140. omniload/src/phantombuster/__init__.py +65 -0
  141. omniload/src/phantombuster/client.py +87 -0
  142. omniload/src/pinterest/__init__.py +82 -0
  143. omniload/src/pipedrive/__init__.py +212 -0
  144. omniload/src/pipedrive/helpers/__init__.py +37 -0
  145. omniload/src/pipedrive/helpers/custom_fields_munger.py +116 -0
  146. omniload/src/pipedrive/helpers/pages.py +129 -0
  147. omniload/src/pipedrive/settings.py +41 -0
  148. omniload/src/pipedrive/typing.py +17 -0
  149. omniload/src/plusvibeai/__init__.py +335 -0
  150. omniload/src/plusvibeai/helpers.py +544 -0
  151. omniload/src/plusvibeai/settings.py +252 -0
  152. omniload/src/primer/__init__.py +45 -0
  153. omniload/src/primer/helpers.py +79 -0
  154. omniload/src/quickbooks/__init__.py +117 -0
  155. omniload/src/reddit_ads/__init__.py +183 -0
  156. omniload/src/reddit_ads/helpers.py +232 -0
  157. omniload/src/resource.py +40 -0
  158. omniload/src/revenuecat/__init__.py +83 -0
  159. omniload/src/revenuecat/helpers.py +237 -0
  160. omniload/src/salesforce/__init__.py +170 -0
  161. omniload/src/salesforce/helpers.py +78 -0
  162. omniload/src/shopify/__init__.py +1953 -0
  163. omniload/src/shopify/exceptions.py +17 -0
  164. omniload/src/shopify/helpers.py +202 -0
  165. omniload/src/shopify/settings.py +19 -0
  166. omniload/src/slack/__init__.py +290 -0
  167. omniload/src/slack/helpers.py +218 -0
  168. omniload/src/slack/settings.py +36 -0
  169. omniload/src/smartsheets/__init__.py +82 -0
  170. omniload/src/snapchat_ads/__init__.py +455 -0
  171. omniload/src/snapchat_ads/client.py +72 -0
  172. omniload/src/snapchat_ads/helpers.py +630 -0
  173. omniload/src/snapchat_ads/settings.py +130 -0
  174. omniload/src/socrata_source/__init__.py +83 -0
  175. omniload/src/socrata_source/helpers.py +85 -0
  176. omniload/src/socrata_source/settings.py +8 -0
  177. omniload/src/solidgate/__init__.py +219 -0
  178. omniload/src/solidgate/helpers.py +154 -0
  179. omniload/src/sources.py +5408 -0
  180. omniload/src/sql_database/__init__.py +0 -0
  181. omniload/src/sql_database/callbacks.py +66 -0
  182. omniload/src/stripe_analytics/__init__.py +183 -0
  183. omniload/src/stripe_analytics/helpers.py +386 -0
  184. omniload/src/stripe_analytics/settings.py +80 -0
  185. omniload/src/table_definition.py +15 -0
  186. omniload/src/testdata/fakebqcredentials.json +14 -0
  187. omniload/src/tiktok_ads/__init__.py +150 -0
  188. omniload/src/tiktok_ads/tiktok_helpers.py +130 -0
  189. omniload/src/time.py +11 -0
  190. omniload/src/trustpilot/__init__.py +48 -0
  191. omniload/src/trustpilot/client.py +48 -0
  192. omniload/src/version.py +6 -0
  193. omniload/src/wise/__init__.py +68 -0
  194. omniload/src/wise/client.py +63 -0
  195. omniload/src/zendesk/__init__.py +480 -0
  196. omniload/src/zendesk/helpers/__init__.py +39 -0
  197. omniload/src/zendesk/helpers/api_helpers.py +119 -0
  198. omniload/src/zendesk/helpers/credentials.py +68 -0
  199. omniload/src/zendesk/helpers/talk_api.py +132 -0
  200. omniload/src/zendesk/settings.py +71 -0
  201. omniload/src/zoom/__init__.py +99 -0
  202. omniload/src/zoom/helpers.py +102 -0
  203. omniload/testdata/.gitignore +2 -0
  204. omniload/testdata/create_replace.csv +21 -0
  205. omniload/testdata/delete_insert_expected.csv +6 -0
  206. omniload/testdata/delete_insert_part1.csv +5 -0
  207. omniload/testdata/delete_insert_part2.csv +6 -0
  208. omniload/testdata/merge_expected.csv +5 -0
  209. omniload/testdata/merge_part1.csv +4 -0
  210. omniload/testdata/merge_part2.csv +5 -0
  211. omniload/tests/unit/test_smartsheets.py +133 -0
  212. omniload-0.0.0.dev0.dist-info/METADATA +439 -0
  213. omniload-0.0.0.dev0.dist-info/RECORD +218 -0
  214. omniload-0.0.0.dev0.dist-info/WHEEL +4 -0
  215. omniload-0.0.0.dev0.dist-info/entry_points.txt +2 -0
  216. omniload-0.0.0.dev0.dist-info/licenses/LICENSE.Apache-2.0 +201 -0
  217. omniload-0.0.0.dev0.dist-info/licenses/LICENSE.md +21 -0
  218. omniload-0.0.0.dev0.dist-info/licenses/NOTICE +35 -0
@@ -0,0 +1,800 @@
1
+ # Copyright 2022-2025 ScaleVector
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ This is a module that provides a DLT source to retrieve data from multiple endpoints of the HubSpot API using a specified API key. The retrieved data is returned as a tuple of Dlt resources, one for each endpoint.
17
+
18
+ The source retrieves data from the following endpoints:
19
+ - CRM Companies
20
+ - CRM Contacts
21
+ - CRM Deals
22
+ - CRM Tickets
23
+ - CRM Products
24
+ - CRM Quotes
25
+ - Web Analytics Events
26
+
27
+ For each endpoint, a resource and transformer function are defined to retrieve data and transform it to a common format.
28
+ The resource functions yield the raw data retrieved from the API, while the transformer functions are used to retrieve
29
+ additional information from the Web Analytics Events endpoint.
30
+
31
+ The source also supports enabling Web Analytics Events for each endpoint by setting the corresponding enable flag to True.
32
+
33
+ Example:
34
+ To retrieve data from all endpoints, use the following code:
35
+
36
+ python
37
+
38
+ >>> resources = hubspot(api_key="your_api_key")
39
+ """
40
+
41
+ from typing import Any, Dict, Iterator, List, Literal, Optional, Sequence
42
+ from urllib.parse import parse_qs, quote, urlparse
43
+
44
+ import dlt
45
+ from dlt.common import pendulum
46
+ from dlt.common.typing import TDataItems
47
+ from dlt.sources import DltResource
48
+
49
+ from .helpers import (
50
+ _get_property_names,
51
+ fetch_data,
52
+ fetch_data_raw,
53
+ fetch_data_search,
54
+ fetch_property_history,
55
+ )
56
+ from .settings import (
57
+ ALL,
58
+ CRM_OBJECT_ENDPOINTS,
59
+ CRM_OWNERS_ENDPOINT,
60
+ CRM_SCHEMAS_ENDPOINT,
61
+ DEFAULT_CALL_PROPS,
62
+ DEFAULT_CART_PROPS,
63
+ DEFAULT_COMMERCE_PAYMENT_PROPS,
64
+ DEFAULT_COMPANY_PROPS,
65
+ DEFAULT_CONTACT_PROPS,
66
+ DEFAULT_DEAL_PROPS,
67
+ DEFAULT_DISCOUNT_PROPS,
68
+ DEFAULT_EMAIL_PROPS,
69
+ DEFAULT_FEE_PROPS,
70
+ DEFAULT_FEEDBACK_SUBMISSION_PROPS,
71
+ DEFAULT_INVOICE_PROPS,
72
+ DEFAULT_LINE_ITEM_PROPS,
73
+ DEFAULT_MEETING_PROPS,
74
+ DEFAULT_NOTE_PROPS,
75
+ DEFAULT_PRODUCT_PROPS,
76
+ DEFAULT_QUOTE_PROPS,
77
+ DEFAULT_TASK_PROPS,
78
+ DEFAULT_TAX_PROPS,
79
+ DEFAULT_TICKET_PROPS,
80
+ OBJECT_TYPE_PLURAL,
81
+ STARTDATE,
82
+ WEB_ANALYTICS_EVENTS_ENDPOINT,
83
+ )
84
+
85
+ THubspotObjectType = Literal[
86
+ "company",
87
+ "contact",
88
+ "deal",
89
+ "ticket",
90
+ "product",
91
+ "quote",
92
+ "call",
93
+ "email",
94
+ "feedback_submission",
95
+ "line_item",
96
+ "meeting",
97
+ "note",
98
+ "task",
99
+ "cart",
100
+ "discount",
101
+ "fee",
102
+ "invoice",
103
+ "commerce_payment",
104
+ "tax",
105
+ ]
106
+
107
+
108
+ def _last_value_to_ms(last_value) -> Optional[str]:
109
+ """Convert dlt incremental last_value (ISO string or datetime) to ms timestamp string."""
110
+ if last_value is None:
111
+ return None
112
+ dt = (
113
+ pendulum.parse(last_value)
114
+ if isinstance(last_value, str)
115
+ else pendulum.instance(last_value)
116
+ )
117
+ return str(int(dt.timestamp() * 1000))
118
+
119
+
120
+ @dlt.source(name="hubspot", max_table_nesting=0)
121
+ def hubspot(
122
+ api_key: str = dlt.secrets.value,
123
+ include_history: bool = False,
124
+ include_custom_props: bool = True,
125
+ custom_object: str = None,
126
+ start_date: Optional[Any] = None,
127
+ end_date: Optional[Any] = None,
128
+ ) -> Sequence[DltResource]:
129
+ """
130
+ A DLT source that retrieves data from the HubSpot API using the
131
+ specified API key.
132
+
133
+ This function retrieves data for several HubSpot API endpoints,
134
+ including companies, contacts, deals, tickets, products and web
135
+ analytics events. It returns a tuple of Dlt resources, one for
136
+ each endpoint.
137
+
138
+ Args:
139
+ api_key (Optional[str]):
140
+ The API key used to authenticate with the HubSpot API. Defaults
141
+ to dlt.secrets.value.
142
+ include_history (Optional[bool]):
143
+ Whether to load history of property changes along with entities.
144
+ The history entries are loaded to separate tables.
145
+
146
+ Returns:
147
+ Sequence[DltResource]: Dlt resources, one for each HubSpot API endpoint.
148
+
149
+ Notes:
150
+ This function uses the `fetch_data` function to retrieve data from the
151
+ HubSpot CRM API. The API key is passed to `fetch_data` as the
152
+ `api_key` argument.
153
+ """
154
+
155
+ if start_date is None:
156
+ start_date = "1970-01-01T00:00:00Z"
157
+ elif not isinstance(start_date, str):
158
+ start_date = pendulum.instance(start_date).to_iso8601_string()
159
+ if end_date is not None and not isinstance(end_date, str):
160
+ end_date = pendulum.instance(end_date).to_iso8601_string()
161
+
162
+ @dlt.resource(
163
+ name="companies", write_disposition="merge", primary_key=["hs_object_id"]
164
+ )
165
+ def companies(
166
+ api_key: str = api_key,
167
+ include_history: bool = include_history,
168
+ include_custom_props: bool = include_custom_props,
169
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
170
+ "hs_lastmodifieddate",
171
+ initial_value=start_date,
172
+ end_value=end_date,
173
+ ),
174
+ ) -> Iterator[TDataItems]:
175
+ """Hubspot companies resource"""
176
+ yield from crm_objects(
177
+ "company",
178
+ api_key,
179
+ include_history=include_history,
180
+ props=DEFAULT_COMPANY_PROPS,
181
+ include_custom_props=include_custom_props,
182
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
183
+ end_date_ms=_last_value_to_ms(end_date),
184
+ )
185
+
186
+ @dlt.resource(
187
+ name="contacts", write_disposition="merge", primary_key=["hs_object_id"]
188
+ )
189
+ def contacts(
190
+ api_key: str = api_key,
191
+ include_history: bool = include_history,
192
+ include_custom_props: bool = include_custom_props,
193
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
194
+ "lastmodifieddate",
195
+ initial_value=start_date,
196
+ end_value=end_date,
197
+ ),
198
+ ) -> Iterator[TDataItems]:
199
+ """Hubspot contacts resource"""
200
+ yield from crm_objects(
201
+ "contact",
202
+ api_key,
203
+ include_history,
204
+ DEFAULT_CONTACT_PROPS,
205
+ include_custom_props,
206
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
207
+ end_date_ms=_last_value_to_ms(end_date),
208
+ )
209
+
210
+ @dlt.resource(name="deals", write_disposition="merge", primary_key=["hs_object_id"])
211
+ def deals(
212
+ api_key: str = api_key,
213
+ include_history: bool = include_history,
214
+ include_custom_props: bool = include_custom_props,
215
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
216
+ "hs_lastmodifieddate",
217
+ initial_value=start_date,
218
+ end_value=end_date,
219
+ ),
220
+ ) -> Iterator[TDataItems]:
221
+ """Hubspot deals resource"""
222
+ yield from crm_objects(
223
+ "deal",
224
+ api_key,
225
+ include_history,
226
+ DEFAULT_DEAL_PROPS,
227
+ include_custom_props,
228
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
229
+ end_date_ms=_last_value_to_ms(end_date),
230
+ )
231
+
232
+ @dlt.resource(
233
+ name="tickets", write_disposition="merge", primary_key=["hs_object_id"]
234
+ )
235
+ def tickets(
236
+ api_key: str = api_key,
237
+ include_history: bool = include_history,
238
+ include_custom_props: bool = include_custom_props,
239
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
240
+ "hs_lastmodifieddate",
241
+ initial_value=start_date,
242
+ end_value=end_date,
243
+ ),
244
+ ) -> Iterator[TDataItems]:
245
+ """Hubspot tickets resource"""
246
+ yield from crm_objects(
247
+ "ticket",
248
+ api_key,
249
+ include_history,
250
+ DEFAULT_TICKET_PROPS,
251
+ include_custom_props,
252
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
253
+ end_date_ms=_last_value_to_ms(end_date),
254
+ )
255
+
256
+ @dlt.resource(
257
+ name="products", write_disposition="merge", primary_key=["hs_object_id"]
258
+ )
259
+ def products(
260
+ api_key: str = api_key,
261
+ include_history: bool = include_history,
262
+ include_custom_props: bool = include_custom_props,
263
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
264
+ "hs_lastmodifieddate",
265
+ initial_value=start_date,
266
+ end_value=end_date,
267
+ ),
268
+ ) -> Iterator[TDataItems]:
269
+ """Hubspot products resource"""
270
+ yield from crm_objects(
271
+ "product",
272
+ api_key,
273
+ include_history,
274
+ DEFAULT_PRODUCT_PROPS,
275
+ include_custom_props,
276
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
277
+ end_date_ms=_last_value_to_ms(end_date),
278
+ )
279
+
280
+ @dlt.resource(name="calls", write_disposition="merge", primary_key=["hs_object_id"])
281
+ def calls(
282
+ api_key: str = api_key,
283
+ include_history: bool = include_history,
284
+ include_custom_props: bool = include_custom_props,
285
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
286
+ "hs_lastmodifieddate",
287
+ initial_value=start_date,
288
+ end_value=end_date,
289
+ ),
290
+ ) -> Iterator[TDataItems]:
291
+ """Hubspot calls resource"""
292
+ yield from crm_objects(
293
+ "call",
294
+ api_key,
295
+ include_history,
296
+ DEFAULT_CALL_PROPS,
297
+ include_custom_props,
298
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
299
+ end_date_ms=_last_value_to_ms(end_date),
300
+ )
301
+
302
+ @dlt.resource(
303
+ name="emails", write_disposition="merge", primary_key=["hs_object_id"]
304
+ )
305
+ def emails(
306
+ api_key: str = api_key,
307
+ include_history: bool = include_history,
308
+ include_custom_props: bool = include_custom_props,
309
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
310
+ "hs_lastmodifieddate",
311
+ initial_value=start_date,
312
+ end_value=end_date,
313
+ ),
314
+ ) -> Iterator[TDataItems]:
315
+ """Hubspot emails resource"""
316
+ yield from crm_objects(
317
+ "email",
318
+ api_key,
319
+ include_history,
320
+ DEFAULT_EMAIL_PROPS,
321
+ include_custom_props,
322
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
323
+ end_date_ms=_last_value_to_ms(end_date),
324
+ )
325
+
326
+ @dlt.resource(
327
+ name="feedback_submissions",
328
+ write_disposition="merge",
329
+ primary_key=["hs_object_id"],
330
+ )
331
+ def feedback_submissions(
332
+ api_key: str = api_key,
333
+ include_history: bool = include_history,
334
+ include_custom_props: bool = include_custom_props,
335
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
336
+ "hs_lastmodifieddate",
337
+ initial_value=start_date,
338
+ end_value=end_date,
339
+ ),
340
+ ) -> Iterator[TDataItems]:
341
+ """Hubspot feedback submissions resource"""
342
+ yield from crm_objects(
343
+ "feedback_submission",
344
+ api_key,
345
+ include_history,
346
+ DEFAULT_FEEDBACK_SUBMISSION_PROPS,
347
+ include_custom_props,
348
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
349
+ end_date_ms=_last_value_to_ms(end_date),
350
+ )
351
+
352
+ @dlt.resource(
353
+ name="line_items", write_disposition="merge", primary_key=["hs_object_id"]
354
+ )
355
+ def line_items(
356
+ api_key: str = api_key,
357
+ include_history: bool = include_history,
358
+ include_custom_props: bool = include_custom_props,
359
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
360
+ "hs_lastmodifieddate",
361
+ initial_value=start_date,
362
+ end_value=end_date,
363
+ ),
364
+ ) -> Iterator[TDataItems]:
365
+ """Hubspot line items resource"""
366
+ yield from crm_objects(
367
+ "line_item",
368
+ api_key,
369
+ include_history,
370
+ DEFAULT_LINE_ITEM_PROPS,
371
+ include_custom_props,
372
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
373
+ end_date_ms=_last_value_to_ms(end_date),
374
+ )
375
+
376
+ @dlt.resource(
377
+ name="meetings", write_disposition="merge", primary_key=["hs_object_id"]
378
+ )
379
+ def meetings(
380
+ api_key: str = api_key,
381
+ include_history: bool = include_history,
382
+ include_custom_props: bool = include_custom_props,
383
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
384
+ "hs_lastmodifieddate",
385
+ initial_value=start_date,
386
+ end_value=end_date,
387
+ ),
388
+ ) -> Iterator[TDataItems]:
389
+ """Hubspot meetings resource"""
390
+ yield from crm_objects(
391
+ "meeting",
392
+ api_key,
393
+ include_history,
394
+ DEFAULT_MEETING_PROPS,
395
+ include_custom_props,
396
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
397
+ end_date_ms=_last_value_to_ms(end_date),
398
+ )
399
+
400
+ @dlt.resource(name="notes", write_disposition="merge", primary_key=["hs_object_id"])
401
+ def notes(
402
+ api_key: str = api_key,
403
+ include_history: bool = include_history,
404
+ include_custom_props: bool = include_custom_props,
405
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
406
+ "hs_lastmodifieddate",
407
+ initial_value=start_date,
408
+ end_value=end_date,
409
+ ),
410
+ ) -> Iterator[TDataItems]:
411
+ """Hubspot notes resource"""
412
+ yield from crm_objects(
413
+ "note",
414
+ api_key,
415
+ include_history,
416
+ DEFAULT_NOTE_PROPS,
417
+ include_custom_props,
418
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
419
+ end_date_ms=_last_value_to_ms(end_date),
420
+ )
421
+
422
+ @dlt.resource(name="tasks", write_disposition="merge", primary_key=["hs_object_id"])
423
+ def tasks(
424
+ api_key: str = api_key,
425
+ include_history: bool = include_history,
426
+ include_custom_props: bool = include_custom_props,
427
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
428
+ "hs_lastmodifieddate",
429
+ initial_value=start_date,
430
+ end_value=end_date,
431
+ ),
432
+ ) -> Iterator[TDataItems]:
433
+ """Hubspot tasks resource"""
434
+ yield from crm_objects(
435
+ "task",
436
+ api_key,
437
+ include_history,
438
+ DEFAULT_TASK_PROPS,
439
+ include_custom_props,
440
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
441
+ end_date_ms=_last_value_to_ms(end_date),
442
+ )
443
+
444
+ @dlt.resource(name="carts", write_disposition="merge", primary_key=["hs_object_id"])
445
+ def carts(
446
+ api_key: str = api_key,
447
+ include_history: bool = include_history,
448
+ include_custom_props: bool = include_custom_props,
449
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
450
+ "hs_lastmodifieddate",
451
+ initial_value=start_date,
452
+ end_value=end_date,
453
+ ),
454
+ ) -> Iterator[TDataItems]:
455
+ """Hubspot carts resource"""
456
+ yield from crm_objects(
457
+ "cart",
458
+ api_key,
459
+ include_history,
460
+ DEFAULT_CART_PROPS,
461
+ include_custom_props,
462
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
463
+ end_date_ms=_last_value_to_ms(end_date),
464
+ )
465
+
466
+ @dlt.resource(
467
+ name="discounts", write_disposition="merge", primary_key=["hs_object_id"]
468
+ )
469
+ def discounts(
470
+ api_key: str = api_key,
471
+ include_history: bool = include_history,
472
+ include_custom_props: bool = include_custom_props,
473
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
474
+ "hs_lastmodifieddate",
475
+ initial_value=start_date,
476
+ end_value=end_date,
477
+ ),
478
+ ) -> Iterator[TDataItems]:
479
+ """Hubspot discounts resource"""
480
+ yield from crm_objects(
481
+ "discount",
482
+ api_key,
483
+ include_history,
484
+ DEFAULT_DISCOUNT_PROPS,
485
+ include_custom_props,
486
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
487
+ end_date_ms=_last_value_to_ms(end_date),
488
+ )
489
+
490
+ @dlt.resource(name="fees", write_disposition="merge", primary_key=["hs_object_id"])
491
+ def fees(
492
+ api_key: str = api_key,
493
+ include_history: bool = include_history,
494
+ include_custom_props: bool = include_custom_props,
495
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
496
+ "hs_lastmodifieddate",
497
+ initial_value=start_date,
498
+ end_value=end_date,
499
+ ),
500
+ ) -> Iterator[TDataItems]:
501
+ """Hubspot fees resource"""
502
+ yield from crm_objects(
503
+ "fee",
504
+ api_key,
505
+ include_history,
506
+ DEFAULT_FEE_PROPS,
507
+ include_custom_props,
508
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
509
+ end_date_ms=_last_value_to_ms(end_date),
510
+ )
511
+
512
+ @dlt.resource(
513
+ name="invoices", write_disposition="merge", primary_key=["hs_object_id"]
514
+ )
515
+ def invoices(
516
+ api_key: str = api_key,
517
+ include_history: bool = include_history,
518
+ include_custom_props: bool = include_custom_props,
519
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
520
+ "hs_lastmodifieddate",
521
+ initial_value=start_date,
522
+ end_value=end_date,
523
+ ),
524
+ ) -> Iterator[TDataItems]:
525
+ """Hubspot invoices resource"""
526
+ yield from crm_objects(
527
+ "invoice",
528
+ api_key,
529
+ include_history,
530
+ DEFAULT_INVOICE_PROPS,
531
+ include_custom_props,
532
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
533
+ end_date_ms=_last_value_to_ms(end_date),
534
+ )
535
+
536
+ @dlt.resource(
537
+ name="commerce_payments",
538
+ write_disposition="merge",
539
+ primary_key=["hs_object_id"],
540
+ )
541
+ def commerce_payments(
542
+ api_key: str = api_key,
543
+ include_history: bool = include_history,
544
+ include_custom_props: bool = include_custom_props,
545
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
546
+ "hs_lastmodifieddate",
547
+ initial_value=start_date,
548
+ end_value=end_date,
549
+ ),
550
+ ) -> Iterator[TDataItems]:
551
+ """Hubspot commerce payments resource"""
552
+ yield from crm_objects(
553
+ "commerce_payment",
554
+ api_key,
555
+ include_history,
556
+ DEFAULT_COMMERCE_PAYMENT_PROPS,
557
+ include_custom_props,
558
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
559
+ end_date_ms=_last_value_to_ms(end_date),
560
+ )
561
+
562
+ @dlt.resource(name="taxes", write_disposition="merge", primary_key=["hs_object_id"])
563
+ def taxes(
564
+ api_key: str = api_key,
565
+ include_history: bool = include_history,
566
+ include_custom_props: bool = include_custom_props,
567
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
568
+ "hs_lastmodifieddate",
569
+ initial_value=start_date,
570
+ end_value=end_date,
571
+ ),
572
+ ) -> Iterator[TDataItems]:
573
+ """Hubspot taxes resource"""
574
+ yield from crm_objects(
575
+ "tax",
576
+ api_key,
577
+ include_history,
578
+ DEFAULT_TAX_PROPS,
579
+ include_custom_props,
580
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
581
+ end_date_ms=_last_value_to_ms(end_date),
582
+ )
583
+
584
+ @dlt.resource(
585
+ name="quotes", write_disposition="merge", primary_key=["hs_object_id"]
586
+ )
587
+ def quotes(
588
+ api_key: str = api_key,
589
+ include_history: bool = include_history,
590
+ include_custom_props: bool = include_custom_props,
591
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
592
+ "hs_lastmodifieddate",
593
+ initial_value=start_date,
594
+ end_value=end_date,
595
+ ),
596
+ ) -> Iterator[TDataItems]:
597
+ """Hubspot quotes resource"""
598
+ yield from crm_objects(
599
+ "quote",
600
+ api_key,
601
+ include_history,
602
+ DEFAULT_QUOTE_PROPS,
603
+ include_custom_props,
604
+ start_date_ms=_last_value_to_ms(updated_at.last_value),
605
+ end_date_ms=_last_value_to_ms(end_date),
606
+ )
607
+
608
+ @dlt.resource(name="owners", write_disposition="merge", primary_key="id")
609
+ def owners(
610
+ api_key: str = api_key,
611
+ ) -> Iterator[TDataItems]:
612
+ """Hubspot owners resource"""
613
+ yield from fetch_data(CRM_OWNERS_ENDPOINT, api_key, resource_name="owners")
614
+
615
+ @dlt.resource(name="schemas", write_disposition="merge", primary_key="id")
616
+ def schemas(
617
+ api_key: str = api_key,
618
+ ) -> Iterator[TDataItems]:
619
+ """Hubspot schemas resource"""
620
+ yield from fetch_data(CRM_SCHEMAS_ENDPOINT, api_key, resource_name="schemas")
621
+
622
+ @dlt.resource(write_disposition="merge", primary_key="hs_object_id")
623
+ def custom(
624
+ api_key: str = api_key,
625
+ custom_object_name: str = custom_object,
626
+ ) -> Iterator[TDataItems]:
627
+ custom_objects = fetch_data_raw(CRM_SCHEMAS_ENDPOINT, api_key)
628
+ object_type_id = None
629
+ associations = None
630
+ if ":" in custom_object_name:
631
+ fields = custom_object_name.split(":")
632
+ if len(fields) == 2:
633
+ custom_object_name = fields[0]
634
+ associations = fields[1]
635
+
636
+ custom_object_lowercase = custom_object_name.lower()
637
+
638
+ for custom_object in custom_objects["results"]:
639
+ if custom_object["name"].lower() == custom_object_lowercase:
640
+ object_type_id = custom_object["objectTypeId"]
641
+ break
642
+
643
+ # sometimes people use the plural name of the object type by accident, we should try to match that if we can
644
+ if "labels" in custom_object:
645
+ if custom_object_lowercase == custom_object["labels"]["plural"].lower():
646
+ object_type_id = custom_object["objectTypeId"]
647
+ break
648
+
649
+ if object_type_id is None:
650
+ raise ValueError(f"There is no such custom object as {custom_object_name}")
651
+ custom_object_properties = f"crm/v3/properties/{object_type_id}"
652
+
653
+ props_pages = fetch_data(custom_object_properties, api_key)
654
+ props = []
655
+ for page in props_pages:
656
+ props.extend([prop["name"] for prop in page])
657
+ props = ",".join(sorted(list(set(props))))
658
+
659
+ custom_object_endpoint = f"crm/v3/objects/{object_type_id}/?properties={props}"
660
+ if associations:
661
+ custom_object_endpoint += f"&associations={associations}"
662
+
663
+ """Hubspot custom object details resource"""
664
+ yield from fetch_data(custom_object_endpoint, api_key, resource_name="custom")
665
+
666
+ return (
667
+ companies,
668
+ contacts,
669
+ deals,
670
+ tickets,
671
+ products,
672
+ quotes,
673
+ calls,
674
+ emails,
675
+ feedback_submissions,
676
+ line_items,
677
+ meetings,
678
+ notes,
679
+ tasks,
680
+ carts,
681
+ discounts,
682
+ fees,
683
+ invoices,
684
+ commerce_payments,
685
+ taxes,
686
+ owners,
687
+ schemas,
688
+ custom,
689
+ )
690
+
691
+
692
+ def crm_objects(
693
+ object_type: str,
694
+ api_key: str = dlt.secrets.value,
695
+ include_history: bool = False,
696
+ props: Sequence[str] = None,
697
+ include_custom_props: bool = True,
698
+ start_date_ms: Optional[str] = None,
699
+ end_date_ms: Optional[str] = None,
700
+ ) -> Iterator[TDataItems]:
701
+ """Building blocks for CRM resources."""
702
+ if props == ALL:
703
+ props = list(_get_property_names(api_key, object_type))
704
+
705
+ if include_custom_props:
706
+ props += _get_property_names(api_key, object_type)
707
+
708
+ props = ",".join(sorted(list(set(props))))
709
+
710
+ if start_date_ms is not None:
711
+ _qs = parse_qs(urlparse(CRM_OBJECT_ENDPOINTS[object_type]).query)
712
+ assoc_types = [t for t in _qs.get("associations", [""])[0].split(",") if t]
713
+ yield from fetch_data_search(
714
+ object_type,
715
+ api_key,
716
+ props,
717
+ start_date_ms,
718
+ end_date_ms=end_date_ms,
719
+ association_types=assoc_types,
720
+ )
721
+ return
722
+
723
+ params = {"properties": props, "limit": 100}
724
+
725
+ yield from fetch_data(CRM_OBJECT_ENDPOINTS[object_type], api_key, params=params)
726
+ if include_history:
727
+ # Get history separately, as requesting both all properties and history together
728
+ # is likely to hit hubspot's URL length limit
729
+ for history_entries in fetch_property_history(
730
+ CRM_OBJECT_ENDPOINTS[object_type],
731
+ api_key,
732
+ props,
733
+ ):
734
+ yield dlt.mark.with_table_name(
735
+ history_entries,
736
+ OBJECT_TYPE_PLURAL[object_type] + "_property_history",
737
+ )
738
+
739
+
740
+ @dlt.resource
741
+ def hubspot_events_for_objects(
742
+ object_type: THubspotObjectType,
743
+ object_ids: List[str],
744
+ api_key: str = dlt.secrets.value,
745
+ start_date: pendulum.DateTime = STARTDATE,
746
+ ) -> DltResource:
747
+ """
748
+ A standalone DLT resources that retrieves web analytics events from the HubSpot API for a particular object type and list of object ids.
749
+
750
+ Args:
751
+ object_type(THubspotObjectType, required): One of the hubspot object types see definition of THubspotObjectType literal
752
+ object_ids: (List[THubspotObjectType], required): List of object ids to track events
753
+ api_key (str, optional): The API key used to authenticate with the HubSpot API. Defaults to dlt.secrets.value.
754
+ start_date (datetime, optional): The initial date time from which start getting events, default to STARTDATE
755
+
756
+ Returns:
757
+ incremental dlt resource to track events for objects from the list
758
+ """
759
+
760
+ end_date = pendulum.now().isoformat()
761
+ name = object_type + "_events"
762
+
763
+ def get_web_analytics_events(
764
+ occurred_at: dlt.sources.incremental[str],
765
+ ) -> Iterator[List[Dict[str, Any]]]:
766
+ """
767
+ A helper function that retrieves web analytics events for a given object type from the HubSpot API.
768
+
769
+ Args:
770
+ object_type (str): The type of object for which to retrieve web analytics events.
771
+
772
+ Yields:
773
+ dict: A dictionary representing a web analytics event.
774
+ """
775
+ for object_id in object_ids:
776
+ yield from fetch_data(
777
+ WEB_ANALYTICS_EVENTS_ENDPOINT.format(
778
+ objectType=object_type,
779
+ objectId=object_id,
780
+ occurredAfter=quote(occurred_at.last_value),
781
+ occurredBefore=quote(end_date),
782
+ ),
783
+ api_key=api_key,
784
+ )
785
+
786
+ return dlt.resource(
787
+ get_web_analytics_events,
788
+ name=name,
789
+ primary_key="id",
790
+ write_disposition="append",
791
+ selected=True,
792
+ table_name=lambda e: name + "_" + str(e["eventType"]),
793
+ )(
794
+ dlt.sources.incremental(
795
+ "occurredAt",
796
+ initial_value=start_date.isoformat(),
797
+ range_end="closed",
798
+ range_start="closed",
799
+ )
800
+ )