ingestr 0.13.35__tar.gz → 0.13.36__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ingestr might be problematic. Click here for more details.

Files changed (267) hide show
  1. ingestr-0.13.36/.gitleaksignore +4 -0
  2. {ingestr-0.13.35 → ingestr-0.13.36}/PKG-INFO +1 -1
  3. ingestr-0.13.36/docs/media/freshdesk_ingestion.png +0 -0
  4. ingestr-0.13.36/docs/media/google_analytics_realtime_report.png +0 -0
  5. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/frankfurter.md +4 -2
  6. ingestr-0.13.36/docs/supported-sources/freshdesk.md +47 -0
  7. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/google_analytics.md +43 -10
  8. ingestr-0.13.36/ingestr/src/buildinfo.py +1 -0
  9. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/factory.py +2 -0
  10. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/frankfurter/__init__.py +44 -36
  11. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/frankfurter/helpers.py +2 -2
  12. ingestr-0.13.36/ingestr/src/freshdesk/__init__.py +72 -0
  13. ingestr-0.13.36/ingestr/src/freshdesk/freshdesk_client.py +102 -0
  14. ingestr-0.13.36/ingestr/src/freshdesk/settings.py +9 -0
  15. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_analytics/__init__.py +21 -3
  16. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_analytics/helpers.py +121 -6
  17. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sources.py +80 -54
  18. ingestr-0.13.35/.gitleaksignore +0 -3
  19. ingestr-0.13.35/ingestr/src/buildinfo.py +0 -1
  20. {ingestr-0.13.35 → ingestr-0.13.36}/.dockerignore +0 -0
  21. {ingestr-0.13.35 → ingestr-0.13.36}/.githooks/pre-commit-hook.sh +0 -0
  22. {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/deploy-docs.yml +0 -0
  23. {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/release.yml +0 -0
  24. {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/secrets-scan.yml +0 -0
  25. {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/tests.yml +0 -0
  26. {ingestr-0.13.35 → ingestr-0.13.36}/.gitignore +0 -0
  27. {ingestr-0.13.35 → ingestr-0.13.36}/.python-version +0 -0
  28. {ingestr-0.13.35 → ingestr-0.13.36}/.vale.ini +0 -0
  29. {ingestr-0.13.35 → ingestr-0.13.36}/Dockerfile +0 -0
  30. {ingestr-0.13.35 → ingestr-0.13.36}/LICENSE.md +0 -0
  31. {ingestr-0.13.35 → ingestr-0.13.36}/Makefile +0 -0
  32. {ingestr-0.13.35 → ingestr-0.13.36}/README.md +0 -0
  33. {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/config.mjs +0 -0
  34. {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/theme/custom.css +0 -0
  35. {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/theme/index.js +0 -0
  36. {ingestr-0.13.35 → ingestr-0.13.36}/docs/commands/example-uris.md +0 -0
  37. {ingestr-0.13.35 → ingestr-0.13.36}/docs/commands/ingest.md +0 -0
  38. {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/core-concepts.md +0 -0
  39. {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/incremental-loading.md +0 -0
  40. {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/quickstart.md +0 -0
  41. {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/telemetry.md +0 -0
  42. {ingestr-0.13.35 → ingestr-0.13.36}/docs/index.md +0 -0
  43. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/applovin_max.png +0 -0
  44. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/athena.png +0 -0
  45. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/clickhouse_img.png +0 -0
  46. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/github.png +0 -0
  47. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/googleanalytics.png +0 -0
  48. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/kinesis.bigquery.png +0 -0
  49. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/linkedin_ads.png +0 -0
  50. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/personio.png +0 -0
  51. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/personio_duckdb.png +0 -0
  52. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/pipedrive.png +0 -0
  53. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/stripe_postgres.png +0 -0
  54. {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/tiktok.png +0 -0
  55. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/adjust.md +0 -0
  56. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/airtable.md +0 -0
  57. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/applovin.md +0 -0
  58. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/applovin_max.md +0 -0
  59. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/appsflyer.md +0 -0
  60. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/appstore.md +0 -0
  61. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/asana.md +0 -0
  62. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/athena.md +0 -0
  63. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/bigquery.md +0 -0
  64. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/chess.md +0 -0
  65. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/clickhouse.md +0 -0
  66. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/csv.md +0 -0
  67. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/custom_queries.md +0 -0
  68. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/databricks.md +0 -0
  69. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/db2.md +0 -0
  70. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/duckdb.md +0 -0
  71. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/dynamodb.md +0 -0
  72. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/facebook-ads.md +0 -0
  73. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gcs.md +0 -0
  74. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/github.md +0 -0
  75. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/google-ads.md +0 -0
  76. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gorgias.md +0 -0
  77. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gsheets.md +0 -0
  78. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/hubspot.md +0 -0
  79. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/kafka.md +0 -0
  80. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/kinesis.md +0 -0
  81. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/klaviyo.md +0 -0
  82. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/linkedin_ads.md +0 -0
  83. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mongodb.md +0 -0
  84. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mssql.md +0 -0
  85. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mysql.md +0 -0
  86. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/notion.md +0 -0
  87. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/oracle.md +0 -0
  88. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/personio.md +0 -0
  89. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/pipedrive.md +0 -0
  90. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/postgres.md +0 -0
  91. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/redshift.md +0 -0
  92. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/s3.md +0 -0
  93. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/salesforce.md +0 -0
  94. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/sap-hana.md +0 -0
  95. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/shopify.md +0 -0
  96. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/slack.md +0 -0
  97. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/snowflake.md +0 -0
  98. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/sqlite.md +0 -0
  99. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/stripe.md +0 -0
  100. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/tiktok-ads.md +0 -0
  101. {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/zendesk.md +0 -0
  102. {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-kinesis-bigquery.md +0 -0
  103. {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-personio-duckdb.md +0 -0
  104. {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-stripe-postgres.md +0 -0
  105. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/conftest.py +0 -0
  106. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/main.py +0 -0
  107. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/.gitignore +0 -0
  108. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/adjust/__init__.py +0 -0
  109. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/adjust/adjust_helpers.py +0 -0
  110. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/airtable/__init__.py +0 -0
  111. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/applovin/__init__.py +0 -0
  112. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/applovin_max/__init__.py +0 -0
  113. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appsflyer/__init__.py +0 -0
  114. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appsflyer/client.py +0 -0
  115. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/__init__.py +0 -0
  116. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/client.py +0 -0
  117. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/errors.py +0 -0
  118. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/models.py +0 -0
  119. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/resources.py +0 -0
  120. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/arrow/__init__.py +0 -0
  121. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/__init__.py +0 -0
  122. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/helpers.py +0 -0
  123. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/settings.py +0 -0
  124. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/blob.py +0 -0
  125. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/__init__.py +0 -0
  126. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/helpers.py +0 -0
  127. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/settings.py +0 -0
  128. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/collector/spinner.py +0 -0
  129. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/destinations.py +0 -0
  130. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/dynamodb/__init__.py +0 -0
  131. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/errors.py +0 -0
  132. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/__init__.py +0 -0
  133. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/exceptions.py +0 -0
  134. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/helpers.py +0 -0
  135. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/settings.py +0 -0
  136. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/__init__.py +0 -0
  137. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/helpers.py +0 -0
  138. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/readers.py +0 -0
  139. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filters.py +0 -0
  140. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/__init__.py +0 -0
  141. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/helpers.py +0 -0
  142. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/queries.py +0 -0
  143. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/settings.py +0 -0
  144. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/__init__.py +0 -0
  145. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/field.py +0 -0
  146. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/metrics.py +0 -0
  147. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/predicates.py +0 -0
  148. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/reports.py +0 -0
  149. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/README.md +0 -0
  150. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/__init__.py +0 -0
  151. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/__init__.py +0 -0
  152. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/api_calls.py +0 -0
  153. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/data_processing.py +0 -0
  154. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/gorgias/__init__.py +0 -0
  155. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/gorgias/helpers.py +0 -0
  156. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/__init__.py +0 -0
  157. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/helpers.py +0 -0
  158. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/settings.py +0 -0
  159. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kafka/__init__.py +0 -0
  160. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kafka/helpers.py +0 -0
  161. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kinesis/__init__.py +0 -0
  162. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kinesis/helpers.py +0 -0
  163. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/__init__.py +0 -0
  164. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/client.py +0 -0
  165. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/helpers.py +0 -0
  166. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/__init__.py +0 -0
  167. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/dimension_time_enum.py +0 -0
  168. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/helpers.py +0 -0
  169. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/loader.py +0 -0
  170. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/mongodb/__init__.py +0 -0
  171. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/mongodb/helpers.py +0 -0
  172. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/__init__.py +0 -0
  173. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/__init__.py +0 -0
  174. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/client.py +0 -0
  175. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/database.py +0 -0
  176. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/settings.py +0 -0
  177. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/partition.py +0 -0
  178. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/personio/__init__.py +0 -0
  179. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/personio/helpers.py +0 -0
  180. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/__init__.py +0 -0
  181. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/__init__.py +0 -0
  182. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/custom_fields_munger.py +0 -0
  183. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/pages.py +0 -0
  184. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/settings.py +0 -0
  185. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/typing.py +0 -0
  186. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/resource.py +0 -0
  187. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/salesforce/__init__.py +0 -0
  188. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/salesforce/helpers.py +0 -0
  189. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/__init__.py +0 -0
  190. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/exceptions.py +0 -0
  191. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/helpers.py +0 -0
  192. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/settings.py +0 -0
  193. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/__init__.py +0 -0
  194. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/helpers.py +0 -0
  195. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/settings.py +0 -0
  196. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sql_database/__init__.py +0 -0
  197. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sql_database/callbacks.py +0 -0
  198. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/__init__.py +0 -0
  199. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/helpers.py +0 -0
  200. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/settings.py +0 -0
  201. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/table_definition.py +0 -0
  202. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/telemetry/event.py +0 -0
  203. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/testdata/fakebqcredentials.json +0 -0
  204. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/tiktok_ads/__init__.py +0 -0
  205. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/tiktok_ads/tiktok_helpers.py +0 -0
  206. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/time.py +0 -0
  207. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/version.py +0 -0
  208. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/__init__.py +0 -0
  209. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/__init__.py +0 -0
  210. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/api_helpers.py +0 -0
  211. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/credentials.py +0 -0
  212. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/talk_api.py +0 -0
  213. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/settings.py +0 -0
  214. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/.gitignore +0 -0
  215. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/create_replace.csv +0 -0
  216. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_expected.csv +0 -0
  217. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_part1.csv +0 -0
  218. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_part2.csv +0 -0
  219. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_expected.csv +0 -0
  220. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_part1.csv +0 -0
  221. {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_part2.csv +0 -0
  222. {ingestr-0.13.35 → ingestr-0.13.36}/package-lock.json +0 -0
  223. {ingestr-0.13.35 → ingestr-0.13.36}/package.json +0 -0
  224. {ingestr-0.13.35 → ingestr-0.13.36}/pyproject.toml +0 -0
  225. {ingestr-0.13.35 → ingestr-0.13.36}/requirements-dev.txt +0 -0
  226. {ingestr-0.13.35 → ingestr-0.13.36}/requirements.in +0 -0
  227. {ingestr-0.13.35 → ingestr-0.13.36}/requirements.txt +0 -0
  228. {ingestr-0.13.35 → ingestr-0.13.36}/requirements_arm64.txt +0 -0
  229. {ingestr-0.13.35 → ingestr-0.13.36}/resources/demo.gif +0 -0
  230. {ingestr-0.13.35 → ingestr-0.13.36}/resources/demo.tape +0 -0
  231. {ingestr-0.13.35 → ingestr-0.13.36}/resources/ingestr.svg +0 -0
  232. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/AMPM.yml +0 -0
  233. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Acronyms.yml +0 -0
  234. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Colons.yml +0 -0
  235. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Contractions.yml +0 -0
  236. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/DateFormat.yml +0 -0
  237. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ellipses.yml +0 -0
  238. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/EmDash.yml +0 -0
  239. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Exclamation.yml +0 -0
  240. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/FirstPerson.yml +0 -0
  241. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Gender.yml +0 -0
  242. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/GenderBias.yml +0 -0
  243. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/HeadingPunctuation.yml +0 -0
  244. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Headings.yml +0 -0
  245. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Latin.yml +0 -0
  246. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/LyHyphens.yml +0 -0
  247. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/OptionalPlurals.yml +0 -0
  248. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ordinal.yml +0 -0
  249. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/OxfordComma.yml +0 -0
  250. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Parens.yml +0 -0
  251. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Passive.yml +0 -0
  252. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Periods.yml +0 -0
  253. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Quotes.yml +0 -0
  254. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ranges.yml +0 -0
  255. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Semicolons.yml +0 -0
  256. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Slang.yml +0 -0
  257. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Spacing.yml +0 -0
  258. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Spelling.yml +0 -0
  259. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Units.yml +0 -0
  260. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/We.yml +0 -0
  261. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Will.yml +0 -0
  262. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/WordList.yml +0 -0
  263. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/meta.json +0 -0
  264. {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/vocab.txt +0 -0
  265. {ingestr-0.13.35 → ingestr-0.13.36}/styles/bruin/Ingestr.yml +0 -0
  266. {ingestr-0.13.35 → ingestr-0.13.36}/styles/config/vocabularies/bruin/accept.txt +0 -0
  267. {ingestr-0.13.35 → ingestr-0.13.36}/test.env.template +0 -0
@@ -0,0 +1,4 @@
1
+ ingestr/src/telemetry/event.py:generic-api-key:17
2
+ ingestr/src/testdata/fakebqcredentials.json:private-key:5
3
+ docs/supported-sources/shopify.md:generic-api-key:26
4
+ ingestr/src/google_analytics/helpers_test.py:generic-api-key:36
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ingestr
3
- Version: 0.13.35
3
+ Version: 0.13.36
4
4
  Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
5
5
  Project-URL: Homepage, https://github.com/bruin-data/ingestr
6
6
  Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -41,7 +41,9 @@ ingestr ingest \
41
41
  ### **`--interval-end` (Optional)**
42
42
  - **Description**: The end date for fetching historical exchange rates.
43
43
  - **Value**: A date in the format `YYYY-MM-DD` (e.g., `'2025-03-28'`).
44
- - **Purpose**: Defines the ending point for fetching historical data. If not provided, it defaults to the value of `--interval-start`.
44
+ - **Purpose**: Defines the end point for fetching historical data.
45
+ - If `--interval-start` is provided without `--interval-end`, `--interval-end` defaults to the current date and retrieves up until the latest published data.
46
+ - If `--interval-end` is provided without `--interval-start`, it will be ignored and the call will retrieve the last published data.
45
47
  - For `latest` and `currencies` this parameter is ignored.
46
48
 
47
49
  ---
@@ -104,7 +106,7 @@ ingestr ingest \
104
106
  - **Primary Key**: Composite key of `date` and `currency_name`.
105
107
  - **Notes**:
106
108
  - An optional start and end date can be added via the arguments `--interval-start` and optionally `--interval-end` to define the date range (see examples below). If no start date is specified, the date will default today's date (and thus return the latest exchange rates).
107
- - If a start date but no end date is specified, then the end date will default to the start date and ingestr will retrieve data for the specified start date only.
109
+ - If a start date but no end date is specified, then the end date will default to today's date and ingestr will retrieve data up until the latest published data.
108
110
  - Note that the [Frankfurter API](https://www.frankfurter.dev/) only publishes updates Monday-Friday. If the given date is on the weekend, the date will default to the previous Friday. The source is however implemented in ingestr in such a way as to avoid duplicating rows of data in this case (see [Incremental Loading - Replace](https://bruin-data.github.io/ingestr/getting-started/incremental-loading.html)).
109
111
 
110
112
  #### **Example Table: Handling Weekend Dates**
@@ -0,0 +1,47 @@
1
+ # Freshdesk
2
+
3
+ [Freshdesk](https://www.freshworks.com/freshdesk/) is a cloud-based customer service platform that helps businesses manages customer support via multiple channels including email,phone,websites, and social media.
4
+
5
+ ingestr supports Freshdesk as a source.
6
+
7
+ ## URI format
8
+
9
+ The URI format for Freshdesk is:
10
+
11
+ ```
12
+ freshdesk://<domain>?api_key=<api_key>
13
+ ```
14
+
15
+ URI parameters:
16
+ - `api_key` : The API token used for authentication with Freshdesk.
17
+ - `domain`: The domain of your Freshdesk account, found in your account URL. For example, if your account URL is https://my_company.freshdesk.com/, then `my_company` is your domain.
18
+
19
+ The URI is used to connect to the Freshdesk API for extracting data.
20
+
21
+ ## Setting up a Freshdesk integration
22
+
23
+ Freshdesk requires a few steps to set up an integration, please follow dltHub [setup guide](https://dlthub.com/docs/dlt-ecosystem/verified-sources/freshdesk#setup-guide).
24
+
25
+ After completing the guide,, you will have your `api_key` and `domain`. For example, if your api_key is
26
+ `test_key` and domain `my_company`, you can use the following command to copy data from Freshdesk into a DuckDB database:
27
+
28
+ ```sh
29
+ ingestr ingest --source-uri 'freshdesk://$DOMAIN?api_key=$API_KEY' --source-table 'contacts' --dest-uri duckdb:///freshdesk.duckdb --dest-table 'dest.contacts'
30
+ ```
31
+ This command copies data from the Freshdesk source to DuckDB.
32
+
33
+ <img alt="freshdesk_img" src="../media/freshdesk_ingestion.png" />
34
+
35
+ ## Available Tables
36
+ The Freshdesk source allows you to ingest the following tables:
37
+
38
+ | S.No. | Name | Description |
39
+ | ----- | --------- | ----------------------------------------------------------------------------------------- |
40
+ | 1. | agents | Users responsible for managing and resolving customer inquiries and support tickets. |
41
+ | 2. | companies | Customer organizations or groups that agents support. |
42
+ | 3. | contacts | Individuals or customers who reach out for support. |
43
+ | 4. | groups | Agents organized based on specific criteria. |
44
+ | 5. | roles | Predefined sets of permissions that determine what actions an agent can perform. |
45
+ | 6. | tickets | Customer inquiries or issues submitted via various channels like email, chat, phone, etc. |
46
+
47
+ Use these as the `--source-table` parameter in the `ingestr ingest` command.
@@ -19,31 +19,64 @@ URI parameters:
19
19
  - `credentials_path`: The path to the service account JSON file.
20
20
  - `property_id`: It is a unique number that identifies a particular property on Google Analytics. [Follow this guide](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) to know more about property ID.
21
21
 
22
- ## Setting up an Google Analytics Integration
22
+ ## Setting up a Google Analytics Integration
23
23
  Google Analytics requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_analytics#grab-google-service-account-credentials). Once you complete the guide, you should have an `.json` file and `project_id`.
24
24
 
25
- ## Table: Custom Reports
26
- Custom reports allow you to retrieve data based on specific `dimensions` and `metrics`.
25
+ ## Available Tables:
26
+ ### Custom reports
27
+ - `Custom reports`: allow you to retrieve data based on specific `dimensions` and `metrics`.
27
28
 
28
- Custom Table Format:
29
+ #### Custom Table Format:
29
30
  ```
30
31
  custom:<dimensions>:<metrics>
31
32
  ```
32
33
 
33
- ### Parameters:
34
- - `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#dimensions) to retrieve.
35
- - `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#metrics) to retrieve.
34
+ #### Parameters:
35
+ - `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#dimensions) to retrieve.
36
+ - `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#metrics) to retrieve.
36
37
 
37
- ### Example
38
+ #### Example
38
39
 
39
40
  ```sh
40
41
  ingestr ingest \
41
42
  --source-uri "googleanalytics://?credentials_path="ingestr/src/g_analytics.json&property_id=id123" \
42
- --source-table "custom:city,date:clicks,activeUsers,newUsers" \
43
+ --source-table "custom:date:activeUsers" \
43
44
  --dest-uri "duckdb:///analytics.duckdb" \
44
45
  --dest-table "dest.custom"
45
46
  ```
46
47
 
47
48
  This command will retrieve report and save it to the `dest.custom` table in the DuckDB database.
48
49
 
49
- <img alt="google_analytics_img" src="../media/googleanalytics.png" />
50
+ <img alt="google_analytics_img" src="../media/googleanalytics.png" />
51
+
52
+
53
+ ### Realtime reports
54
+ `Realtime reports`: allows you to retrieve data based on specific `dimensions`, `metrics`, with optional `minutes_ranges`.
55
+
56
+ #### Realtime Report Table Format:
57
+ ```
58
+ realtime:<dimensions>:<metrics>
59
+
60
+ ```
61
+ ```
62
+ realtime:<dimensions>:<metrics>:<minutes_ranges>
63
+ ```
64
+
65
+ #### Parameters:
66
+ - `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#dimensions) to retrieve.
67
+ - `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#metrics) to retrieve.
68
+ - `minutes_ranges`(optional): Allows you to specify time windows for retrieving data. You can define up to two time ranges in your query, formatted as comma-separated values (e.g., "0-5,25-29"). Each range represents minutes in the past from the current time.
69
+ If no minute_ranges are specified, the system defaults to retrieving data from the last 30 minutes. For more information read [here](https://developers.google.com/analytics/devguides/reporting/data/v1/realtime-basics#minute_ranges)
70
+
71
+ #### Example
72
+
73
+ ```sh
74
+ ingestr ingest \
75
+ --source-uri "googleanalytics://?credentials_path="ingestr/src/g_analytics.json&property_id=id123" \
76
+ --source-table "realtime:streamId:activeUsers:0-4,10-29" \
77
+ --dest-uri "duckdb:///analytics.duckdb" \
78
+ --dest-table "dest.realtime"
79
+ ```
80
+ This command will retrieve report and save it to the `dest.realtime` table in the DuckDB database.
81
+
82
+ <img alt="google_analytics_realtime_report_img" src="../media/google_analytics_realtime_report.png"/>
@@ -0,0 +1 @@
1
+ version = "v0.13.36"
@@ -53,6 +53,7 @@ from ingestr.src.sources import (
53
53
  StripeAnalyticsSource,
54
54
  TikTokSource,
55
55
  ZendeskSource,
56
+ FreshdeskSource,
56
57
  )
57
58
 
58
59
  SQL_SOURCE_SCHEMES = [
@@ -148,6 +149,7 @@ class SourceDestinationFactory:
148
149
  "kinesis": KinesisSource,
149
150
  "pipedrive": PipedriveSource,
150
151
  "frankfurter": FrankfurterSource,
152
+ "freshdesk": FreshdeskSource,
151
153
  }
152
154
  destinations: Dict[str, Type[DestinationProtocol]] = {
153
155
  "bigquery": BigQueryDestination,
@@ -1,4 +1,4 @@
1
- from typing import Any, Iterator, Optional
1
+ from typing import Any, Iterator
2
2
 
3
3
  import dlt
4
4
  from dlt.common.pendulum import pendulum
@@ -13,25 +13,28 @@ from ingestr.src.frankfurter.helpers import get_path_with_retry
13
13
  max_table_nesting=0,
14
14
  )
15
15
  def frankfurter_source(
16
- table: str,
17
- start_date: Optional[TAnyDateTime] = None,
18
- end_date: Optional[TAnyDateTime] = None,
16
+ start_date: TAnyDateTime,
17
+ end_date: TAnyDateTime,
19
18
  ) -> Any:
20
19
  """
21
20
  A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
22
21
  various types of data: currencies, latest rates, historical rates.
23
-
24
- Returns the appropriate resource based on the provided parameters.
25
22
  """
26
- # Determine which resource to return based on the `table` parameter
27
- if table == "currencies":
28
- return currencies()
23
+ date_time = dlt.sources.incremental(
24
+
25
+ "date",
26
+ initial_value=start_date,
27
+ end_value=end_date,
28
+ range_start="closed",
29
+ range_end="closed",
30
+ )
29
31
 
30
- elif table == "latest":
31
- return latest()
32
+ return (
33
+ currencies(),
34
+ latest(),
35
+ exchange_rates(start_date=date_time, end_date=end_date),
32
36
 
33
- elif table == "exchange_rates":
34
- return exchange_rates(start_date=start_date, end_date=end_date)
37
+ )
35
38
 
36
39
 
37
40
  @dlt.resource(
@@ -53,13 +56,13 @@ def currencies() -> Iterator[dict]:
53
56
 
54
57
 
55
58
  @dlt.resource(
56
- write_disposition="replace",
59
+ write_disposition="merge",
57
60
  columns={
58
61
  "date": {"data_type": "text"},
59
- "currency_name": {"data_type": "text"},
62
+ "currency_code": {"data_type": "text"},
60
63
  "rate": {"data_type": "double"},
61
64
  },
62
- primary_key=["date", "currency_name"], # Composite primary key
65
+ primary_key=["date", "currency_code"], # Composite primary key
63
66
  )
64
67
  def latest() -> Iterator[dict]:
65
68
  """
@@ -69,50 +72,54 @@ def latest() -> Iterator[dict]:
69
72
  url = "latest?"
70
73
 
71
74
  # Fetch data
72
- latest_data = get_path_with_retry(url)
75
+ data = get_path_with_retry(url)
73
76
 
74
77
  # Extract rates and base currency
75
- rates = latest_data["rates"]
78
+ rates = data["rates"]
76
79
 
77
- # Prepare the date
78
- date = pendulum.now().to_date_string()
80
+ date = pendulum.parse(data["date"])
79
81
 
80
82
  # Add the base currency (EUR) with a rate of 1.0
81
83
  yield {
82
84
  "date": date,
83
- "currency_name": "EUR",
85
+ "currency_code": "EUR",
84
86
  "rate": 1.0,
85
87
  }
86
88
 
87
89
  # Add all currencies and their rates
88
- for currency_name, rate in rates.items():
90
+ for currency_code, rate in rates.items():
89
91
  yield {
90
92
  "date": date,
91
- "currency_name": currency_name,
93
+ "currency_code": currency_code,
92
94
  "rate": rate,
93
95
  }
94
96
 
95
97
 
96
98
  @dlt.resource(
97
- write_disposition="replace",
99
+ write_disposition="merge",
98
100
  columns={
99
101
  "date": {"data_type": "text"},
100
- "currency_name": {"data_type": "text"},
102
+ "currency_code": {"data_type": "text"},
101
103
  "rate": {"data_type": "double"},
102
104
  },
103
- primary_key=["date", "currency_name"], # Composite primary key
105
+ primary_key=("date", "currency_code"), # Composite primary key
104
106
  )
105
107
  def exchange_rates(
106
- start_date: TAnyDateTime,
107
108
  end_date: TAnyDateTime,
109
+ start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
108
110
  ) -> Iterator[dict]:
109
111
  """
110
112
  Fetches exchange rates for a specified date range.
111
- If only start_date is provided, fetches data for that date.
113
+ If only start_date is provided, fetches data until now.
112
114
  If both start_date and end_date are provided, fetches data for each day in the range.
113
115
  """
114
- start_date_str = ensure_pendulum_datetime(start_date).format("YYYY-MM-DD")
115
- end_date_str = ensure_pendulum_datetime(end_date).format("YYYY-MM-DD")
116
+ # Ensure start_date.last_value is a pendulum.DateTime object
117
+ start_date_obj = ensure_pendulum_datetime(start_date.last_value) # type: ignore
118
+ start_date_str = start_date_obj.format("YYYY-MM-DD")
119
+
120
+ # Ensure end_date is a pendulum.DateTime object
121
+ end_date_obj = ensure_pendulum_datetime(end_date)
122
+ end_date_str = end_date_obj.format("YYYY-MM-DD")
116
123
 
117
124
  # Compose the URL
118
125
  url = f"{start_date_str}..{end_date_str}?"
@@ -121,22 +128,23 @@ def exchange_rates(
121
128
  data = get_path_with_retry(url)
122
129
 
123
130
  # Extract base currency and rates from the API response
124
- base_currency = data["base"]
125
131
  rates = data["rates"]
126
132
 
127
133
  # Iterate over the rates dictionary (one entry per date)
128
134
  for date, daily_rates in rates.items():
135
+ formatted_date = pendulum.parse(date)
136
+
129
137
  # Add the base currency with a rate of 1.0
130
138
  yield {
131
- "date": date,
132
- "currency_name": base_currency,
139
+ "date": formatted_date,
140
+ "currency_code": "EUR",
133
141
  "rate": 1.0,
134
142
  }
135
143
 
136
144
  # Add all other currencies and their rates
137
- for currency_name, rate in daily_rates.items():
145
+ for currency_code, rate in daily_rates.items():
138
146
  yield {
139
- "date": date,
140
- "currency_name": currency_name,
147
+ "date": formatted_date,
148
+ "currency_code": currency_code,
141
149
  "rate": rate,
142
150
  }
@@ -8,7 +8,7 @@ FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
8
8
 
9
9
 
10
10
  def get_url_with_retry(url: str) -> StrAny:
11
- r = requests.get(url)
11
+ r = requests.get(url, timeout=5)
12
12
  return r.json() # type: ignore
13
13
 
14
14
 
@@ -19,7 +19,7 @@ def get_path_with_retry(path: str) -> StrAny:
19
19
  def validate_dates(start_date: datetime, end_date: datetime) -> None:
20
20
  current_date = pendulum.now()
21
21
 
22
- # Check if start_date is in the future
22
+ # Check if start_date is in the futurep
23
23
  if start_date > current_date:
24
24
  raise ValueError("Interval-start cannot be in the future.")
25
25
 
@@ -0,0 +1,72 @@
1
+ """This source uses Freshdesk API and dlt to load data such as Agents, Companies, Tickets
2
+ etc. to the database"""
3
+
4
+ from typing import Any, Dict, Generator, Iterable, List, Optional
5
+
6
+ import dlt
7
+ from dlt.sources import DltResource
8
+
9
+ from .freshdesk_client import FreshdeskClient
10
+ from .settings import DEFAULT_ENDPOINTS
11
+
12
+
13
+ @dlt.source()
14
+ def freshdesk_source(
15
+ endpoints: Optional[List[str]] = None,
16
+ per_page: int = 100,
17
+ domain: str = dlt.secrets.value,
18
+ api_secret_key: str = dlt.secrets.value,
19
+ ) -> Iterable[DltResource]:
20
+ """
21
+ Retrieves data from specified Freshdesk API endpoints.
22
+
23
+ This source supports pagination and incremental data loading. It fetches data from a list of
24
+ specified endpoints, or defaults to predefined endpoints in 'settings.py'.
25
+
26
+ Args:
27
+ endpoints: A list of Freshdesk API endpoints to fetch. Deafults to 'settings.py'.
28
+ per_page: The number of items to fetch per page, with a maximum of 100.
29
+ domain: The Freshdesk domain from which to fetch the data. Defaults to 'config.toml'.
30
+ api_secret_key: Freshdesk API key. Defaults to 'secrets.toml'.
31
+
32
+ Yields:
33
+ Iterable[DltResource]: Resources with data updated after the last 'updated_at'
34
+ timestamp for each endpoint.
35
+ """
36
+ # Instantiate FreshdeskClient with the provided domain and API key
37
+ freshdesk = FreshdeskClient(api_key=api_secret_key, domain=domain)
38
+
39
+ def incremental_resource(
40
+ endpoint: str,
41
+ updated_at: Optional[Any] = dlt.sources.incremental(
42
+ "updated_at", initial_value="2022-01-01T00:00:00Z"
43
+ ),
44
+ ) -> Generator[Dict[Any, Any], Any, None]:
45
+ """
46
+ Fetches and yields paginated data from a specified API endpoint.
47
+ Each page of data is fetched based on the `updated_at` timestamp
48
+ to ensure incremental loading.
49
+ """
50
+
51
+ # Retrieve the last updated timestamp to fetch only new or updated records.
52
+ if updated_at is not None:
53
+ updated_at = updated_at.last_value
54
+
55
+ # Use the FreshdeskClient instance to fetch paginated responses
56
+ yield from freshdesk.paginated_response(
57
+ endpoint=endpoint,
58
+ per_page=per_page,
59
+ updated_at=updated_at,
60
+ )
61
+
62
+ # Set default endpoints if not provided
63
+ endpoints = endpoints or DEFAULT_ENDPOINTS
64
+
65
+ # For each endpoint, create and yield a DLT resource
66
+ for endpoint in endpoints:
67
+ yield dlt.resource(
68
+ incremental_resource,
69
+ name=endpoint,
70
+ write_disposition="merge",
71
+ primary_key="id",
72
+ )(endpoint=endpoint)
@@ -0,0 +1,102 @@
1
+ """Freshdesk Client for making authenticated requests"""
2
+
3
+ import logging
4
+ import time
5
+ from typing import Any, Dict, Iterable, Optional
6
+
7
+ from dlt.common.typing import TDataItem
8
+ from dlt.sources.helpers import requests
9
+
10
+
11
+ class FreshdeskClient:
12
+ """
13
+ Client for making authenticated requests to the Freshdesk API. It incorporates API requests with
14
+ rate limit and pagination.
15
+
16
+ Attributes:
17
+ api_key (str): The API key used for authenticating requests to the Freshdesk API.
18
+ domain (str): The Freshdesk domain specific to the user, used in constructing the base URL.
19
+ base_url (str): The base URL constructed from the domain, targeting the Freshdesk API v2.
20
+ """
21
+
22
+ def __init__(self, api_key: str, domain: str):
23
+ # Initialize the FreshdeskClient instance with API key and domain.
24
+ # The API key is used for authentication with the Freshdesk API.
25
+ # The domain specifies the unique Freshdesk domain of the user.
26
+
27
+ # Store the API key provided during initialization.
28
+ self.api_key = api_key
29
+ # Store the Freshdesk domain provided during initialization.
30
+ self.domain = domain
31
+
32
+ # Construct the base URL for the API requests.
33
+ # This URL is formed by appending the domain to the standard Freshdesk API base URL format.
34
+ # All API requests will use this base URL as their starting point.
35
+ self.base_url = f"https://{domain}.freshdesk.com/api/v2"
36
+
37
+ def _request_with_rate_limit(self, url: str, **kwargs: Any) -> requests.Response:
38
+ """
39
+ Handles rate limits in HTTP requests and ensures
40
+ that the client doesn't exceed the limit set by the server.
41
+ """
42
+
43
+ while True:
44
+ try:
45
+ response = requests.get(url, **kwargs, auth=(self.api_key, "X"))
46
+ response.raise_for_status()
47
+
48
+ return response
49
+ except requests.HTTPError as e:
50
+ if e.response.status_code == 429:
51
+ # Get the 'Retry-After' header to know how long to wait
52
+ # Fallback to 60 seconds if header is missing
53
+ seconds_to_wait = int(e.response.headers.get("Retry-After", 60))
54
+ # Log a warning message
55
+ logging.warning(
56
+ "Rate limited. Waiting to retry after: %s secs", seconds_to_wait
57
+ )
58
+
59
+ # Wait for the specified number of seconds before retrying
60
+ time.sleep(seconds_to_wait)
61
+ else:
62
+ # If the error is not a rate limit (429), raise the exception to be
63
+ # handled elsewhere or stop execution
64
+ raise
65
+
66
+ def paginated_response(
67
+ self,
68
+ endpoint: str,
69
+ per_page: int,
70
+ updated_at: Optional[str] = None,
71
+ ) -> Iterable[TDataItem]:
72
+ """
73
+ Fetches a paginated response from a specified endpoint.
74
+
75
+ This method will continuously fetch data from the given endpoint,
76
+ page by page, until no more data is available or until it reaches data
77
+ updated at the specified timestamp.
78
+ """
79
+ page = 1
80
+ while True:
81
+ # Construct the URL for the specific endpoint
82
+ url = f"{self.base_url}/{endpoint}"
83
+
84
+ params: Dict[str, Any] = {"per_page": per_page, "page": page}
85
+
86
+ # Implement date range splitting logic here, if applicable
87
+ if endpoint in ["tickets", "contacts"]:
88
+ param_key = (
89
+ "updated_since" if endpoint == "tickets" else "_updated_since"
90
+ )
91
+ if updated_at:
92
+ params[param_key] = updated_at
93
+
94
+ # Handle requests with rate-limiting
95
+ # A maximum of 300 pages (30000 tickets) will be returned.
96
+ response = self._request_with_rate_limit(url, params=params)
97
+ data = response.json()
98
+
99
+ if not data:
100
+ break # Stop if no data or max page limit reached
101
+ yield data
102
+ page += 1
@@ -0,0 +1,9 @@
1
+ """
2
+ This module defines default settings for the Freshdesk integration.
3
+
4
+ It specifies a list of default endpoints to be used when interacting with the Freshdesk API,
5
+ covering common entities such as agents, companies, contacts, groups, roles, and tickets.
6
+ """
7
+
8
+ # Define default endpoints for the Freshdesk API integration.
9
+ DEFAULT_ENDPOINTS = ["agents", "companies", "contacts", "groups", "roles", "tickets"]
@@ -13,9 +13,10 @@ from google.analytics.data_v1beta import BetaAnalyticsDataClient
13
13
  from google.analytics.data_v1beta.types import (
14
14
  Dimension,
15
15
  Metric,
16
+ MinuteRange,
16
17
  )
17
18
 
18
- from .helpers import get_report
19
+ from .helpers import get_realtime_report, get_report
19
20
 
20
21
 
21
22
  @dlt.source(max_table_nesting=0)
@@ -29,6 +30,7 @@ def google_analytics(
29
30
  start_date: Optional[pendulum.DateTime] = pendulum.datetime(2024, 1, 1),
30
31
  end_date: Optional[pendulum.DateTime] = None,
31
32
  rows_per_page: int = 10000,
33
+ minute_range_objects: List[MinuteRange] | None = None,
32
34
  ) -> List[DltResource]:
33
35
  try:
34
36
  property_id = int(property_id)
@@ -58,7 +60,7 @@ def google_analytics(
58
60
  dimensions = query["dimensions"]
59
61
 
60
62
  @dlt.resource(
61
- name="basic_report",
63
+ name="custom",
62
64
  merge_key=datetime_dimension,
63
65
  write_disposition="merge",
64
66
  )
@@ -87,6 +89,22 @@ def google_analytics(
87
89
  end_date=end_date,
88
90
  )
89
91
 
92
+ # real time report
93
+ @dlt.resource(
94
+ name="realtime",
95
+ merge_key="ingested_at",
96
+ write_disposition="merge",
97
+ )
98
+ def real_time_report() -> Iterator[TDataItem]:
99
+ yield from get_realtime_report(
100
+ client=client,
101
+ property_id=property_id,
102
+ dimension_list=[Dimension(name=dimension) for dimension in dimensions],
103
+ metric_list=[Metric(name=metric) for metric in query["metrics"]],
104
+ per_page=rows_per_page,
105
+ minute_range_objects=minute_range_objects,
106
+ )
107
+
90
108
  # res = dlt.resource(
91
109
  # basic_report, name="basic_report", merge_key=datetime_dimension, write_disposition="merge"
92
110
  # )(
@@ -103,4 +121,4 @@ def google_analytics(
103
121
  # ),
104
122
  # )
105
123
 
106
- return [basic_report]
124
+ return [basic_report, real_time_report]