ingestr 0.13.10__tar.gz → 0.13.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ingestr might be problematic. Click here for more details.

Files changed (232) hide show
  1. {ingestr-0.13.10 → ingestr-0.13.11}/PKG-INFO +17 -11
  2. {ingestr-0.13.10 → ingestr-0.13.11}/README.md +5 -0
  3. {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/config.mjs +1 -0
  4. ingestr-0.13.11/docs/supported-sources/salesforce.md +68 -0
  5. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/applovin/__init__.py +8 -7
  6. ingestr-0.13.11/ingestr/src/buildinfo.py +1 -0
  7. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/factory.py +2 -0
  8. ingestr-0.13.11/ingestr/src/salesforce/__init__.py +149 -0
  9. ingestr-0.13.11/ingestr/src/salesforce/helpers.py +64 -0
  10. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sources.py +71 -6
  11. {ingestr-0.13.10 → ingestr-0.13.11}/requirements.txt +11 -12
  12. ingestr-0.13.10/ingestr/src/buildinfo.py +0 -1
  13. {ingestr-0.13.10 → ingestr-0.13.11}/.dockerignore +0 -0
  14. {ingestr-0.13.10 → ingestr-0.13.11}/.githooks/pre-commit-hook.sh +0 -0
  15. {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/deploy-docs.yml +0 -0
  16. {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/release.yml +0 -0
  17. {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/secrets-scan.yml +0 -0
  18. {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/tests.yml +0 -0
  19. {ingestr-0.13.10 → ingestr-0.13.11}/.gitignore +0 -0
  20. {ingestr-0.13.10 → ingestr-0.13.11}/.gitleaksignore +0 -0
  21. {ingestr-0.13.10 → ingestr-0.13.11}/.python-version +0 -0
  22. {ingestr-0.13.10 → ingestr-0.13.11}/.vale.ini +0 -0
  23. {ingestr-0.13.10 → ingestr-0.13.11}/Dockerfile +0 -0
  24. {ingestr-0.13.10 → ingestr-0.13.11}/LICENSE.md +0 -0
  25. {ingestr-0.13.10 → ingestr-0.13.11}/Makefile +0 -0
  26. {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/theme/custom.css +0 -0
  27. {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/theme/index.js +0 -0
  28. {ingestr-0.13.10 → ingestr-0.13.11}/docs/commands/example-uris.md +0 -0
  29. {ingestr-0.13.10 → ingestr-0.13.11}/docs/commands/ingest.md +0 -0
  30. {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/core-concepts.md +0 -0
  31. {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/incremental-loading.md +0 -0
  32. {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/quickstart.md +0 -0
  33. {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/telemetry.md +0 -0
  34. {ingestr-0.13.10 → ingestr-0.13.11}/docs/index.md +0 -0
  35. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/applovin_max.png +0 -0
  36. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/athena.png +0 -0
  37. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/clickhouse_img.png +0 -0
  38. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/github.png +0 -0
  39. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/googleanalytics.png +0 -0
  40. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/linkedin_ads.png +0 -0
  41. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/personio.png +0 -0
  42. {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/tiktok.png +0 -0
  43. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/adjust.md +0 -0
  44. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/airtable.md +0 -0
  45. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/applovin.md +0 -0
  46. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/applovin_max.md +0 -0
  47. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/appsflyer.md +0 -0
  48. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/appstore.md +0 -0
  49. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/asana.md +0 -0
  50. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/athena.md +0 -0
  51. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/bigquery.md +0 -0
  52. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/chess.md +0 -0
  53. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/clickhouse.md +0 -0
  54. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/csv.md +0 -0
  55. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/custom_queries.md +0 -0
  56. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/databricks.md +0 -0
  57. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/duckdb.md +0 -0
  58. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/dynamodb.md +0 -0
  59. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/facebook-ads.md +0 -0
  60. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gcs.md +0 -0
  61. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/github.md +0 -0
  62. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/google-ads.md +0 -0
  63. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/google_analytics.md +0 -0
  64. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gorgias.md +0 -0
  65. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gsheets.md +0 -0
  66. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/hubspot.md +0 -0
  67. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/kafka.md +0 -0
  68. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/klaviyo.md +0 -0
  69. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/linkedin_ads.md +0 -0
  70. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mongodb.md +0 -0
  71. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mssql.md +0 -0
  72. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mysql.md +0 -0
  73. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/notion.md +0 -0
  74. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/oracle.md +0 -0
  75. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/personio.md +0 -0
  76. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/postgres.md +0 -0
  77. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/redshift.md +0 -0
  78. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/s3.md +0 -0
  79. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/sap-hana.md +0 -0
  80. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/shopify.md +0 -0
  81. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/slack.md +0 -0
  82. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/snowflake.md +0 -0
  83. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/sqlite.md +0 -0
  84. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/stripe.md +0 -0
  85. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/tiktok-ads.md +0 -0
  86. {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/zendesk.md +0 -0
  87. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/main.py +0 -0
  88. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/.gitignore +0 -0
  89. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/adjust/__init__.py +0 -0
  90. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/adjust/adjust_helpers.py +0 -0
  91. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/airtable/__init__.py +0 -0
  92. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/applovin_max/__init__.py +0 -0
  93. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appsflyer/_init_.py +0 -0
  94. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appsflyer/client.py +0 -0
  95. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/__init__.py +0 -0
  96. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/client.py +0 -0
  97. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/errors.py +0 -0
  98. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/models.py +0 -0
  99. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/resources.py +0 -0
  100. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/arrow/__init__.py +0 -0
  101. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/__init__.py +0 -0
  102. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/helpers.py +0 -0
  103. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/settings.py +0 -0
  104. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/blob.py +0 -0
  105. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/__init__.py +0 -0
  106. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/helpers.py +0 -0
  107. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/settings.py +0 -0
  108. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/destinations.py +0 -0
  109. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/dynamodb/__init__.py +0 -0
  110. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/errors.py +0 -0
  111. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/__init__.py +0 -0
  112. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/exceptions.py +0 -0
  113. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/helpers.py +0 -0
  114. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/settings.py +0 -0
  115. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/__init__.py +0 -0
  116. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/helpers.py +0 -0
  117. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/readers.py +0 -0
  118. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filters.py +0 -0
  119. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/__init__.py +0 -0
  120. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/helpers.py +0 -0
  121. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/queries.py +0 -0
  122. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/settings.py +0 -0
  123. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/__init__.py +0 -0
  124. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/field.py +0 -0
  125. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/metrics.py +0 -0
  126. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/predicates.py +0 -0
  127. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/reports.py +0 -0
  128. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_analytics/__init__.py +0 -0
  129. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_analytics/helpers.py +0 -0
  130. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/README.md +0 -0
  131. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/__init__.py +0 -0
  132. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/__init__.py +0 -0
  133. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/api_calls.py +0 -0
  134. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/data_processing.py +0 -0
  135. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/gorgias/__init__.py +0 -0
  136. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/gorgias/helpers.py +0 -0
  137. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/__init__.py +0 -0
  138. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/helpers.py +0 -0
  139. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/settings.py +0 -0
  140. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/kafka/__init__.py +0 -0
  141. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/kafka/helpers.py +0 -0
  142. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/_init_.py +0 -0
  143. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/client.py +0 -0
  144. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/helpers.py +0 -0
  145. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/__init__.py +0 -0
  146. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/dimension_time_enum.py +0 -0
  147. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/helpers.py +0 -0
  148. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/loader.py +0 -0
  149. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/mongodb/__init__.py +0 -0
  150. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/mongodb/helpers.py +0 -0
  151. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/__init__.py +0 -0
  152. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/__init__.py +0 -0
  153. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/client.py +0 -0
  154. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/database.py +0 -0
  155. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/settings.py +0 -0
  156. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/personio/__init__.py +0 -0
  157. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/personio/helpers.py +0 -0
  158. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/__init__.py +0 -0
  159. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/exceptions.py +0 -0
  160. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/helpers.py +0 -0
  161. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/settings.py +0 -0
  162. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/__init__.py +0 -0
  163. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/helpers.py +0 -0
  164. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/settings.py +0 -0
  165. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sql_database/__init__.py +0 -0
  166. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sql_database/callbacks.py +0 -0
  167. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/__init__.py +0 -0
  168. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/helpers.py +0 -0
  169. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/settings.py +0 -0
  170. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/table_definition.py +0 -0
  171. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/telemetry/event.py +0 -0
  172. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/testdata/fakebqcredentials.json +0 -0
  173. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/tiktok_ads/__init__.py +0 -0
  174. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/tiktok_ads/tiktok_helpers.py +0 -0
  175. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/time.py +0 -0
  176. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/version.py +0 -0
  177. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/__init__.py +0 -0
  178. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/__init__.py +0 -0
  179. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/api_helpers.py +0 -0
  180. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/credentials.py +0 -0
  181. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/talk_api.py +0 -0
  182. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/settings.py +0 -0
  183. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/.gitignore +0 -0
  184. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/create_replace.csv +0 -0
  185. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_expected.csv +0 -0
  186. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_part1.csv +0 -0
  187. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_part2.csv +0 -0
  188. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_expected.csv +0 -0
  189. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_part1.csv +0 -0
  190. {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_part2.csv +0 -0
  191. {ingestr-0.13.10 → ingestr-0.13.11}/package-lock.json +0 -0
  192. {ingestr-0.13.10 → ingestr-0.13.11}/package.json +0 -0
  193. {ingestr-0.13.10 → ingestr-0.13.11}/pyproject.toml +0 -0
  194. {ingestr-0.13.10 → ingestr-0.13.11}/requirements-dev.txt +0 -0
  195. {ingestr-0.13.10 → ingestr-0.13.11}/resources/demo.gif +0 -0
  196. {ingestr-0.13.10 → ingestr-0.13.11}/resources/demo.tape +0 -0
  197. {ingestr-0.13.10 → ingestr-0.13.11}/resources/ingestr.svg +0 -0
  198. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/AMPM.yml +0 -0
  199. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Acronyms.yml +0 -0
  200. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Colons.yml +0 -0
  201. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Contractions.yml +0 -0
  202. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/DateFormat.yml +0 -0
  203. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ellipses.yml +0 -0
  204. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/EmDash.yml +0 -0
  205. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Exclamation.yml +0 -0
  206. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/FirstPerson.yml +0 -0
  207. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Gender.yml +0 -0
  208. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/GenderBias.yml +0 -0
  209. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/HeadingPunctuation.yml +0 -0
  210. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Headings.yml +0 -0
  211. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Latin.yml +0 -0
  212. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/LyHyphens.yml +0 -0
  213. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/OptionalPlurals.yml +0 -0
  214. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ordinal.yml +0 -0
  215. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/OxfordComma.yml +0 -0
  216. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Parens.yml +0 -0
  217. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Passive.yml +0 -0
  218. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Periods.yml +0 -0
  219. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Quotes.yml +0 -0
  220. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ranges.yml +0 -0
  221. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Semicolons.yml +0 -0
  222. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Slang.yml +0 -0
  223. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Spacing.yml +0 -0
  224. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Spelling.yml +0 -0
  225. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Units.yml +0 -0
  226. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/We.yml +0 -0
  227. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Will.yml +0 -0
  228. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/WordList.yml +0 -0
  229. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/meta.json +0 -0
  230. {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/vocab.txt +0 -0
  231. {ingestr-0.13.10 → ingestr-0.13.11}/styles/bruin/Ingestr.yml +0 -0
  232. {ingestr-0.13.10 → ingestr-0.13.11}/styles/config/vocabularies/bruin/accept.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ingestr
3
- Version: 0.13.10
3
+ Version: 0.13.11
4
4
  Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
5
5
  Project-URL: Homepage, https://github.com/bruin-data/ingestr
6
6
  Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -18,42 +18,43 @@ Requires-Dist: asana==3.2.3
18
18
  Requires-Dist: clickhouse-connect==0.8.14
19
19
  Requires-Dist: clickhouse-driver==0.2.9
20
20
  Requires-Dist: clickhouse-sqlalchemy==0.2.7
21
- Requires-Dist: confluent-kafka>=2.6.1
21
+ Requires-Dist: confluent-kafka>=2.8.0
22
22
  Requires-Dist: databricks-sql-connector==2.9.3
23
23
  Requires-Dist: dataclasses-json==0.6.7
24
- Requires-Dist: dlt==1.5.0
25
- Requires-Dist: duckdb-engine==0.13.5
26
- Requires-Dist: duckdb==1.1.3
24
+ Requires-Dist: dlt==1.6.1
25
+ Requires-Dist: duckdb-engine==0.15.0
26
+ Requires-Dist: duckdb==1.2.0
27
27
  Requires-Dist: facebook-business==20.0.0
28
28
  Requires-Dist: flatten-json==0.1.14
29
29
  Requires-Dist: gcsfs==2024.10.0
30
30
  Requires-Dist: google-ads==25.1.0
31
- Requires-Dist: google-analytics-data==0.18.16
31
+ Requires-Dist: google-analytics-data==0.18.17
32
32
  Requires-Dist: google-api-python-client==2.130.0
33
33
  Requires-Dist: google-cloud-bigquery-storage==2.24.0
34
- Requires-Dist: mysql-connector-python==9.1.0
34
+ Requires-Dist: mysql-connector-python==9.2.0
35
35
  Requires-Dist: pendulum==3.0.0
36
36
  Requires-Dist: psutil==6.1.1
37
37
  Requires-Dist: psycopg2-binary==2.9.10
38
38
  Requires-Dist: py-machineid==0.6.0
39
39
  Requires-Dist: pyairtable==2.3.3
40
40
  Requires-Dist: pyarrow==18.1.0
41
- Requires-Dist: pyathena==3.9.0
42
- Requires-Dist: pymongo==4.10.1
41
+ Requires-Dist: pyathena==3.12.2
42
+ Requires-Dist: pymongo==4.11.1
43
43
  Requires-Dist: pymysql==1.1.1
44
44
  Requires-Dist: pyrate-limiter==3.7.0
45
45
  Requires-Dist: redshift-connector==2.1.5
46
46
  Requires-Dist: rich==13.9.4
47
47
  Requires-Dist: rudder-sdk-python==2.1.4
48
48
  Requires-Dist: s3fs==2024.10.0
49
+ Requires-Dist: simple-salesforce==1.12.6
49
50
  Requires-Dist: snowflake-sqlalchemy==1.6.1
50
- Requires-Dist: sqlalchemy-bigquery==1.12.0
51
+ Requires-Dist: sqlalchemy-bigquery==1.12.1
51
52
  Requires-Dist: sqlalchemy-hana==2.0.0
52
53
  Requires-Dist: sqlalchemy-redshift==0.8.14
53
54
  Requires-Dist: sqlalchemy2-stubs==0.0.2a38
54
55
  Requires-Dist: sqlalchemy==1.4.52
55
56
  Requires-Dist: stripe==10.7.0
56
- Requires-Dist: tqdm==4.67.0
57
+ Requires-Dist: tqdm==4.67.1
57
58
  Requires-Dist: typer==0.13.1
58
59
  Requires-Dist: types-requests==2.32.0.20240907
59
60
  Provides-Extra: odbc
@@ -312,6 +313,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
312
313
  <td>✅</td>
313
314
  <td>-</td>
314
315
  </tr>
316
+ <tr>
317
+ <td>Salesforce</td>
318
+ <td>✅</td>
319
+ <td>-</td>
320
+ </tr>
315
321
  <tr>
316
322
  <td>Shopify</td>
317
323
  <td>✅</td>
@@ -248,6 +248,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
248
248
  <td>✅</td>
249
249
  <td>-</td>
250
250
  </tr>
251
+ <tr>
252
+ <td>Salesforce</td>
253
+ <td>✅</td>
254
+ <td>-</td>
255
+ </tr>
251
256
  <tr>
252
257
  <td>Shopify</td>
253
258
  <td>✅</td>
@@ -117,6 +117,7 @@ export default defineConfig({
117
117
  { text: "Notion", link: "/supported-sources/notion.md" },
118
118
  { text: "Personio", link: "/supported-sources/personio.md" },
119
119
  { text: "S3", link: "/supported-sources/s3.md" },
120
+ { text: "Salesforce", link: "/supported-sources/salesforce.md" },
120
121
  { text: "Shopify", link: "/supported-sources/shopify.md" },
121
122
  { text: "Slack", link: "/supported-sources/slack.md" },
122
123
  { text: "Stripe", link: "/supported-sources/stripe.md" },
@@ -0,0 +1,68 @@
1
+ # Salesforce
2
+ [Salesforce](https://www.salesforce.com/) is a cloud-based customer relationship management (CRM) platform that helps businesses manage sales, customer interactions, and business processes. It provides tools for sales automation, customer service, marketing, analytics, and application development.
3
+
4
+ Ingestr supports Salesforce as a source.
5
+
6
+ ## URI format
7
+
8
+ The URI format for Salesforce is as follows:
9
+ ```
10
+ salesforce://?username=<username>&password=<password>&token=<token>
11
+ ```
12
+
13
+ URI parameters:
14
+ - `username` is your Salesforce account username.
15
+ - `password` is your Salesforce account password.
16
+ - `token` is your Salesforce security token.
17
+
18
+ You can obtain your security token by logging into your Salesforce account and navigating to the user settings under "Reset My Security Token."
19
+
20
+ ## Setting up a Salesforce Integration
21
+
22
+ You can obtain an OAuth access token by setting up a connected app in Salesforce and using OAuth 2.0 authentication. For more information, see [Salesforce API Authentication](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm).
23
+
24
+ ## Example
25
+
26
+ Let's say:
27
+ * Your Salesforce username is `user`.
28
+ * Your password is `password123`.
29
+ * Your security token is `fake_token`.
30
+ * You want to ingest `account` data from your salesforce account
31
+ * You want to save this data in a duckdb database `sf.db` under the table `public.account`
32
+
33
+ You can run the following command to achieve this:
34
+ ```sh
35
+ ingestr ingest \
36
+ --source-uri "salesforce://?username=user&password=password123&token=fake_token" \
37
+ --source-table "account" \
38
+ --dest-uri "duckdb:///sf.db" \
39
+ --dest-table "public.account"
40
+ ```
41
+
42
+ ## Tables
43
+
44
+ Salesforce source allows ingesting the following objects into separate tables:
45
+
46
+ | **Table** | **Mode** | **Description** |
47
+ |---------------------------|-----------|----------------|
48
+ | `user` | replace | Refers to an individual who has access to a Salesforce org or instance. |
49
+ | `user_role` | replace | A standard object that represents a role within the organization's hierarchy. |
50
+ | `opportunity` | merge | Represents a sales opportunity for a specific account or contact. |
51
+ | `opportunity_line_item` | merge | Represents individual line items or products associated with an Opportunity. |
52
+ | `opportunity_contact_role` | merge | Represents the association between an Opportunity and a Contact. |
53
+ | `account` | merge | Individual or organization that interacts with your business. |
54
+ | `contact` | replace | An individual person associated with an account or organization. |
55
+ | `lead` | replace | Prospective customer/individual/org. that has shown interest in a company's products/services. |
56
+ | `campaign` | replace | Marketing initiative or project designed to achieve specific goals, such as generating leads. |
57
+ | `campaign_member` | merge | Association between a Contact or Lead and a Campaign. |
58
+ | `product` | replace | For managing and organizing your product-related data within the Salesforce ecosystem. |
59
+ | `pricebook` | replace | Used to manage product pricing and create price books. |
60
+ | `pricebook_entry` | replace | Represents a specific price for a product in a price book. |
61
+ | `task` | merge | Used to track and manage various activities and tasks within the Salesforce platform. |
62
+ | `event` | merge | Used to track and manage calendar-based events, such as meetings, appointments, or calls. |
63
+
64
+ Use these as `--source-table` parameters in the `ingestr ingest` command.
65
+
66
+ > [!WARNING]
67
+ > Salesforce API limits may affect the frequency and volume of data ingestion. Incremental loading is supported for objects with the `SystemModstamp` field, but some objects may require full-refresh loads. This is indicated by `mode` in the tables above. Tables with mode `replace` don't support incremental loads, while the ones with `merge` do.
68
+
@@ -1,10 +1,10 @@
1
- from datetime import datetime, timezone, timedelta
1
+ from datetime import datetime, timedelta, timezone
2
2
  from enum import Enum
3
3
  from typing import Dict, List, Optional
4
- from requests import Response
5
4
 
6
5
  import dlt
7
6
  from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
7
+ from requests import Response
8
8
 
9
9
 
10
10
  class InvalidCustomReportError(Exception):
@@ -13,9 +13,11 @@ class InvalidCustomReportError(Exception):
13
13
  "Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
14
14
  )
15
15
 
16
+
16
17
  class ClientError(Exception):
17
18
  pass
18
19
 
20
+
19
21
  TYPE_HINTS = {
20
22
  "application_is_hidden": {"data_type": "bool"},
21
23
  "average_cpa": {"data_type": "double"},
@@ -119,7 +121,6 @@ def applovin_source(
119
121
  end_date: Optional[str],
120
122
  custom: Optional[str],
121
123
  ):
122
-
123
124
  backfill = False
124
125
  if end_date is None:
125
126
  backfill = True
@@ -127,7 +128,7 @@ def applovin_source(
127
128
  # use the greatest of yesterday and start_date
128
129
  end_date = max(
129
130
  datetime.now(timezone.utc) - timedelta(days=1),
130
- datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
131
+ datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc),
131
132
  ).strftime("%Y-%m-%d")
132
133
 
133
134
  config: RESTAPIConfig = {
@@ -157,7 +158,7 @@ def applovin_source(
157
158
  "paginator": "single_page",
158
159
  "response_actions": [
159
160
  http_error_handler,
160
- ]
161
+ ],
161
162
  },
162
163
  },
163
164
  "resources": [
@@ -177,8 +178,7 @@ def applovin_source(
177
178
  "advertiser-probabilistic-report",
178
179
  "probabilisticReport",
179
180
  exclude(
180
- REPORT_SCHEMA[ReportType.ADVERTISER],
181
- PROBABILISTIC_REPORT_EXCLUDE
181
+ REPORT_SCHEMA[ReportType.ADVERTISER], PROBABILISTIC_REPORT_EXCLUDE
182
182
  ),
183
183
  ReportType.ADVERTISER,
184
184
  ),
@@ -256,6 +256,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
256
256
  def build_type_hints(cols: List[str]) -> dict:
257
257
  return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
258
258
 
259
+
259
260
  def http_error_handler(resp: Response):
260
261
  if not resp.ok:
261
262
  raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")
@@ -0,0 +1 @@
1
+ version = "v0.13.11"
@@ -42,6 +42,7 @@ from ingestr.src.sources import (
42
42
  MongoDbSource,
43
43
  NotionSource,
44
44
  S3Source,
45
+ SalesforceSource,
45
46
  ShopifySource,
46
47
  SlackSource,
47
48
  SqlSource,
@@ -137,6 +138,7 @@ class SourceDestinationFactory:
137
138
  "linkedinads": LinkedInAdsSource,
138
139
  "applovin": AppLovinSource,
139
140
  "applovinmax": ApplovinMaxSource,
141
+ "salesforce": SalesforceSource,
140
142
  "personio": PersonioSource,
141
143
  }
142
144
  destinations: Dict[str, Type[DestinationProtocol]] = {
@@ -0,0 +1,149 @@
1
+ from typing import Iterable
2
+
3
+ import dlt
4
+ from dlt.common.typing import TDataItem
5
+ from dlt.sources import DltResource, incremental
6
+ from simple_salesforce import Salesforce
7
+
8
+ from .helpers import get_records
9
+
10
+
11
+ @dlt.source(name="salesforce")
12
+ def salesforce_source(
13
+ username: str,
14
+ password: str,
15
+ token: str,
16
+ ) -> Iterable[DltResource]:
17
+ """
18
+ Retrieves data from Salesforce using the Salesforce API.
19
+
20
+ Args:
21
+ username (str): The username for authentication.
22
+ password (str): The password for authentication.
23
+ token (str): The security token for authentication.
24
+
25
+ Yields:
26
+ DltResource: Data resources from Salesforce.
27
+ """
28
+
29
+ client = Salesforce(username, password, token)
30
+
31
+ # define resources
32
+ @dlt.resource(write_disposition="replace")
33
+ def user() -> Iterable[TDataItem]:
34
+ yield get_records(client, "User")
35
+
36
+ @dlt.resource(write_disposition="replace")
37
+ def user_role() -> Iterable[TDataItem]:
38
+ yield get_records(client, "UserRole")
39
+
40
+ @dlt.resource(write_disposition="merge")
41
+ def opportunity(
42
+ last_timestamp: incremental[str] = dlt.sources.incremental(
43
+ "SystemModstamp", initial_value=None
44
+ ),
45
+ ) -> Iterable[TDataItem]:
46
+ yield get_records(
47
+ client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
48
+ )
49
+
50
+ @dlt.resource(write_disposition="merge")
51
+ def opportunity_line_item(
52
+ last_timestamp: incremental[str] = dlt.sources.incremental(
53
+ "SystemModstamp", initial_value=None
54
+ ),
55
+ ) -> Iterable[TDataItem]:
56
+ yield get_records(
57
+ client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
58
+ )
59
+
60
+ @dlt.resource(write_disposition="merge")
61
+ def opportunity_contact_role(
62
+ last_timestamp: incremental[str] = dlt.sources.incremental(
63
+ "SystemModstamp", initial_value=None
64
+ ),
65
+ ) -> Iterable[TDataItem]:
66
+ yield get_records(
67
+ client,
68
+ "OpportunityContactRole",
69
+ last_timestamp.last_value,
70
+ "SystemModstamp",
71
+ )
72
+
73
+ @dlt.resource(write_disposition="merge")
74
+ def account(
75
+ last_timestamp: incremental[str] = dlt.sources.incremental(
76
+ "LastModifiedDate", initial_value=None
77
+ ),
78
+ ) -> Iterable[TDataItem]:
79
+ yield get_records(
80
+ client, "Account", last_timestamp.last_value, "LastModifiedDate"
81
+ )
82
+
83
+ @dlt.resource(write_disposition="replace")
84
+ def contact() -> Iterable[TDataItem]:
85
+ yield get_records(client, "Contact")
86
+
87
+ @dlt.resource(write_disposition="replace")
88
+ def lead() -> Iterable[TDataItem]:
89
+ yield get_records(client, "Lead")
90
+
91
+ @dlt.resource(write_disposition="replace")
92
+ def campaign() -> Iterable[TDataItem]:
93
+ yield get_records(client, "Campaign")
94
+
95
+ @dlt.resource(write_disposition="merge")
96
+ def campaign_member(
97
+ last_timestamp: incremental[str] = dlt.sources.incremental(
98
+ "SystemModstamp", initial_value=None
99
+ ),
100
+ ) -> Iterable[TDataItem]:
101
+ yield get_records(
102
+ client, "CampaignMember", last_timestamp.last_value, "SystemModstamp"
103
+ )
104
+
105
+ @dlt.resource(write_disposition="replace")
106
+ def product() -> Iterable[TDataItem]:
107
+ yield get_records(client, "Product2")
108
+
109
+ @dlt.resource(write_disposition="replace")
110
+ def pricebook() -> Iterable[TDataItem]:
111
+ yield get_records(client, "Pricebook2")
112
+
113
+ @dlt.resource(write_disposition="replace")
114
+ def pricebook_entry() -> Iterable[TDataItem]:
115
+ yield get_records(client, "PricebookEntry")
116
+
117
+ @dlt.resource(write_disposition="merge")
118
+ def task(
119
+ last_timestamp: incremental[str] = dlt.sources.incremental(
120
+ "SystemModstamp", initial_value=None
121
+ ),
122
+ ) -> Iterable[TDataItem]:
123
+ yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
124
+
125
+ @dlt.resource(write_disposition="merge")
126
+ def event(
127
+ last_timestamp: incremental[str] = dlt.sources.incremental(
128
+ "SystemModstamp", initial_value=None
129
+ ),
130
+ ) -> Iterable[TDataItem]:
131
+ yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
132
+
133
+ return (
134
+ user,
135
+ user_role,
136
+ opportunity,
137
+ opportunity_line_item,
138
+ opportunity_contact_role,
139
+ account,
140
+ contact,
141
+ lead,
142
+ campaign,
143
+ campaign_member,
144
+ product,
145
+ pricebook,
146
+ pricebook_entry,
147
+ task,
148
+ event,
149
+ )
@@ -0,0 +1,64 @@
1
+ """Salesforce source helpers"""
2
+
3
+ from typing import Iterable, Optional
4
+
5
+ import pendulum
6
+ from dlt.common.typing import TDataItem
7
+ from simple_salesforce import Salesforce
8
+
9
+
10
+ def get_records(
11
+ sf: Salesforce,
12
+ sobject: str,
13
+ last_state: Optional[str] = None,
14
+ replication_key: Optional[str] = None,
15
+ ) -> Iterable[TDataItem]:
16
+ """
17
+ Retrieves records from Salesforce for a specified sObject.
18
+
19
+ Args:
20
+ sf (Salesforce): An instance of the Salesforce API client.
21
+ sobject (str): The name of the sObject to retrieve records from.
22
+ last_state (str, optional): The last known state for incremental loading. Defaults to None.
23
+ replication_key (str, optional): The replication key for incremental loading. Defaults to None.
24
+
25
+ Yields:
26
+ Dict[TDataItem]: A dictionary representing a record from the Salesforce sObject.
27
+ """
28
+
29
+ # Get all fields for the sobject
30
+ desc = getattr(sf, sobject).describe()
31
+ # Salesforce returns compound fields as separate fields, so we need to filter them out
32
+ compound_fields = {
33
+ f["compoundFieldName"]
34
+ for f in desc["fields"]
35
+ if f["compoundFieldName"] is not None
36
+ } - {"Name"}
37
+ # Salesforce returns datetime fields as timestamps, so we need to convert them
38
+ date_fields = {
39
+ f["name"] for f in desc["fields"] if f["type"] in ("datetime",) and f["name"]
40
+ }
41
+ # If no fields are specified, use all fields except compound fields
42
+ fields = [f["name"] for f in desc["fields"] if f["name"] not in compound_fields]
43
+
44
+ # Generate a predicate to filter records by the replication key
45
+ predicate, order_by, n_records = "", "", 0
46
+ if replication_key:
47
+ if last_state:
48
+ predicate = f"WHERE {replication_key} > {last_state}"
49
+ order_by = f"ORDER BY {replication_key} ASC"
50
+ query = f"SELECT {', '.join(fields)} FROM {sobject} {predicate} {order_by}"
51
+
52
+ # Query all records in batches
53
+ for page in getattr(sf.bulk, sobject).query_all(query, lazy_operation=True):
54
+ for record in page:
55
+ # Strip out the attributes field
56
+ record.pop("attributes", None)
57
+ for field in date_fields:
58
+ # Convert Salesforce timestamps to ISO 8601
59
+ if record.get(field):
60
+ record[field] = pendulum.from_timestamp(
61
+ record[field] / 1000,
62
+ ).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
63
+ yield from page
64
+ n_records += len(page)
@@ -15,7 +15,7 @@ from typing import (
15
15
  Optional,
16
16
  Union,
17
17
  )
18
- from urllib.parse import ParseResult, parse_qs, quote, urlparse
18
+ from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
19
19
 
20
20
  import dlt
21
21
  import gcsfs # type: ignore
@@ -84,6 +84,7 @@ from ingestr.src.linkedin_ads.dimension_time_enum import (
84
84
  from ingestr.src.mongodb import mongodb_collection
85
85
  from ingestr.src.notion import notion_databases
86
86
  from ingestr.src.personio import personio_source
87
+ from ingestr.src.salesforce import salesforce_source
87
88
  from ingestr.src.shopify import shopify_source
88
89
  from ingestr.src.slack import slack_source
89
90
  from ingestr.src.sql_database.callbacks import (
@@ -135,10 +136,46 @@ class SqlSource:
135
136
  if uri.startswith("mysql://"):
136
137
  uri = uri.replace("mysql://", "mysql+pymysql://")
137
138
 
139
+ # clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
138
140
  if uri.startswith("clickhouse://"):
139
- uri = uri.replace("clickhouse://", "clickhouse+native://")
140
- if "secure=" not in uri:
141
- uri += "?secure=1"
141
+ parsed_uri = urlparse(uri)
142
+
143
+ username = parsed_uri.username
144
+ if not username:
145
+ raise ValueError(
146
+ "A username is required to connect to the ClickHouse database."
147
+ )
148
+
149
+ password = parsed_uri.password
150
+ if not password:
151
+ raise ValueError(
152
+ "A password is required to authenticate with the ClickHouse database."
153
+ )
154
+
155
+ host = parsed_uri.hostname
156
+ if not host:
157
+ raise ValueError(
158
+ "The hostname or IP address of the ClickHouse server is required to establish a connection."
159
+ )
160
+
161
+ port = parsed_uri.port
162
+ if not port:
163
+ raise ValueError(
164
+ "The TCP port of the ClickHouse server is required to establish a connection."
165
+ )
166
+
167
+ query_params = parse_qs(parsed_uri.query)
168
+
169
+ if "http_port" in query_params:
170
+ del query_params["http_port"]
171
+
172
+ if "secure" not in query_params:
173
+ query_params["secure"] = ["1"]
174
+
175
+ uri = parsed_uri._replace(
176
+ scheme="clickhouse+native",
177
+ query=urlencode(query_params, doseq=True),
178
+ ).geturl()
142
179
 
143
180
  query_adapters = []
144
181
  if kwargs.get("sql_limit"):
@@ -1754,7 +1791,7 @@ class AppLovinSource:
1754
1791
  def dlt_source(self, uri: str, table: str, **kwargs):
1755
1792
  if kwargs.get("incremental_key") is not None:
1756
1793
  raise ValueError(
1757
- "Google Ads takes care of incrementality on its own, you should not provide incremental_key"
1794
+ "Applovin takes care of incrementality on its own, you should not provide incremental_key"
1758
1795
  )
1759
1796
 
1760
1797
  parsed_uri = urlparse(uri)
@@ -1836,6 +1873,34 @@ class ApplovinMaxSource:
1836
1873
  ).with_resources(table)
1837
1874
 
1838
1875
 
1876
+ class SalesforceSource:
1877
+ def handles_incrementality(self) -> bool:
1878
+ return True
1879
+
1880
+ def dlt_source(self, uri: str, table: str, **kwargs):
1881
+ if kwargs.get("incremental_key"):
1882
+ raise ValueError(
1883
+ "Salesforce takes care of incrementality on its own, you should not provide incremental_key"
1884
+ )
1885
+
1886
+ params = parse_qs(urlparse(uri).query)
1887
+ creds = {
1888
+ "username": params.get("username", [None])[0],
1889
+ "password": params.get("password", [None])[0],
1890
+ "token": params.get("token", [None])[0],
1891
+ }
1892
+ for k, v in creds.items():
1893
+ if v is None:
1894
+ raise MissingValueError(k, "Salesforce")
1895
+
1896
+ src = salesforce_source(**creds) # type: ignore
1897
+
1898
+ if table not in src.resources:
1899
+ raise UnsupportedResourceError(table, "Salesforce")
1900
+
1901
+ return src.with_resources(table)
1902
+
1903
+
1839
1904
  class PersonioSource:
1840
1905
  def handles_incrementality(self) -> bool:
1841
1906
  return True
@@ -1874,7 +1939,7 @@ class PersonioSource:
1874
1939
  "custom_reports_list",
1875
1940
  ]:
1876
1941
  raise UnsupportedResourceError(table, "Personio")
1877
-
1942
+
1878
1943
  return personio_source(
1879
1944
  client_id=client_id[0],
1880
1945
  client_secret=client_secret[0],
@@ -1,24 +1,23 @@
1
1
  asana==3.2.3
2
- confluent-kafka>=2.6.1
2
+ confluent-kafka>=2.8.0
3
3
  databricks-sql-connector==2.9.3
4
4
  dataclasses-json==0.6.7
5
- dlt==1.5.0
6
- duckdb_engine==0.13.5
7
- duckdb==1.1.3
5
+ dlt==1.6.1
6
+ duckdb_engine==0.15.0
7
+ duckdb==1.2.0
8
8
  google-ads==25.1.0
9
9
  facebook-business==20.0.0
10
- google-analytics-data==0.18.16
11
10
  google-api-python-client==2.130.0
12
11
  google-cloud-bigquery-storage==2.24.0
13
- mysql-connector-python==9.1.0
12
+ mysql-connector-python==9.2.0
14
13
  pendulum==3.0.0
15
14
  psutil==6.1.1
16
15
  psycopg2-binary==2.9.10
17
16
  py-machineid==0.6.0
18
17
  pyairtable==2.3.3
19
18
  pyarrow==18.1.0
20
- pyathena==3.9.0
21
- pymongo==4.10.1
19
+ pyathena==3.12.2
20
+ pymongo==4.11.1
22
21
  pymysql==1.1.1
23
22
  pyrate-limiter==3.7.0
24
23
  redshift-connector==2.1.5
@@ -26,19 +25,18 @@ rich==13.9.4
26
25
  rudder-sdk-python==2.1.4
27
26
  s3fs==2024.10.0
28
27
  snowflake-sqlalchemy==1.6.1
29
- sqlalchemy-bigquery==1.12.0
28
+ sqlalchemy-bigquery==1.12.1
30
29
  sqlalchemy-hana==2.0.0
31
30
  sqlalchemy-redshift==0.8.14
32
31
  SQLAlchemy==1.4.52
33
32
  sqlalchemy2-stubs==0.0.2a38
34
33
  stripe==10.7.0
35
- tqdm==4.67.0
34
+ tqdm==4.67.1
36
35
  typer==0.13.1
37
36
  types-requests==2.32.0.20240907
38
37
  s3fs==2024.10.0
39
38
  pyarrow==18.1.0
40
- pyathena==3.9.0
41
- google-analytics-data==0.18.16
39
+ google-analytics-data==0.18.17
42
40
  asana==3.2.3
43
41
  flatten_json==0.1.14
44
42
  dataclasses-json==0.6.7
@@ -46,3 +44,4 @@ gcsfs==2024.10.0
46
44
  clickhouse-connect==0.8.14
47
45
  clickhouse-driver==0.2.9
48
46
  clickhouse-sqlalchemy==0.2.7
47
+ simple-salesforce==1.12.6
@@ -1 +0,0 @@
1
- version = "v0.13.10"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes