ingestr 0.13.35__tar.gz → 0.13.36__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr-0.13.36/.gitleaksignore +4 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/PKG-INFO +1 -1
- ingestr-0.13.36/docs/media/freshdesk_ingestion.png +0 -0
- ingestr-0.13.36/docs/media/google_analytics_realtime_report.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/frankfurter.md +4 -2
- ingestr-0.13.36/docs/supported-sources/freshdesk.md +47 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/google_analytics.md +43 -10
- ingestr-0.13.36/ingestr/src/buildinfo.py +1 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/factory.py +2 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/frankfurter/__init__.py +44 -36
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/frankfurter/helpers.py +2 -2
- ingestr-0.13.36/ingestr/src/freshdesk/__init__.py +72 -0
- ingestr-0.13.36/ingestr/src/freshdesk/freshdesk_client.py +102 -0
- ingestr-0.13.36/ingestr/src/freshdesk/settings.py +9 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_analytics/__init__.py +21 -3
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_analytics/helpers.py +121 -6
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sources.py +80 -54
- ingestr-0.13.35/.gitleaksignore +0 -3
- ingestr-0.13.35/ingestr/src/buildinfo.py +0 -1
- {ingestr-0.13.35 → ingestr-0.13.36}/.dockerignore +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.githooks/pre-commit-hook.sh +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/deploy-docs.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/release.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/secrets-scan.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.github/workflows/tests.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.gitignore +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.python-version +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/.vale.ini +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/Dockerfile +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/LICENSE.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/Makefile +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/README.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/config.mjs +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/theme/custom.css +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/.vitepress/theme/index.js +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/commands/example-uris.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/commands/ingest.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/core-concepts.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/incremental-loading.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/quickstart.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/getting-started/telemetry.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/index.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/applovin_max.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/athena.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/clickhouse_img.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/github.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/googleanalytics.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/kinesis.bigquery.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/linkedin_ads.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/personio.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/personio_duckdb.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/pipedrive.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/stripe_postgres.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/media/tiktok.png +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/adjust.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/airtable.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/applovin.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/applovin_max.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/appsflyer.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/appstore.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/asana.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/athena.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/bigquery.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/chess.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/clickhouse.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/csv.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/custom_queries.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/databricks.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/db2.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/duckdb.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/dynamodb.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/facebook-ads.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gcs.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/github.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/google-ads.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gorgias.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/gsheets.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/hubspot.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/kafka.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/kinesis.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/klaviyo.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/linkedin_ads.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mongodb.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mssql.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/mysql.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/notion.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/oracle.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/personio.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/pipedrive.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/postgres.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/redshift.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/s3.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/salesforce.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/sap-hana.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/shopify.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/slack.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/snowflake.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/sqlite.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/stripe.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/tiktok-ads.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/supported-sources/zendesk.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-kinesis-bigquery.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-personio-duckdb.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/docs/tutorials/load-stripe-postgres.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/conftest.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/main.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/.gitignore +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/adjust/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/adjust/adjust_helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/airtable/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/applovin/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/applovin_max/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appsflyer/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appsflyer/client.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/client.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/errors.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/models.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/appstore/resources.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/arrow/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/asana_source/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/blob.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/chess/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/collector/spinner.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/destinations.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/dynamodb/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/errors.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/exceptions.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/facebook_ads/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filesystem/readers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/filters.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/queries.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/github/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/field.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/metrics.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/predicates.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_ads/reports.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/README.md +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/api_calls.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/google_sheets/helpers/data_processing.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/gorgias/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/gorgias/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/hubspot/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kafka/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kafka/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kinesis/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/kinesis/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/client.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/klaviyo/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/dimension_time_enum.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/linkedin_ads/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/loader.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/mongodb/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/mongodb/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/client.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/helpers/database.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/notion/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/partition.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/personio/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/personio/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/custom_fields_munger.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/helpers/pages.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/pipedrive/typing.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/resource.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/salesforce/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/salesforce/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/exceptions.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/shopify/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/slack/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sql_database/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/sql_database/callbacks.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/stripe_analytics/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/table_definition.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/telemetry/event.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/testdata/fakebqcredentials.json +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/tiktok_ads/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/tiktok_ads/tiktok_helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/time.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/version.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/__init__.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/api_helpers.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/credentials.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/helpers/talk_api.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/src/zendesk/settings.py +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/.gitignore +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/create_replace.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_expected.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_part1.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/delete_insert_part2.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_expected.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_part1.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/ingestr/testdata/merge_part2.csv +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/package-lock.json +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/package.json +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/pyproject.toml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/requirements-dev.txt +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/requirements.in +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/requirements.txt +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/requirements_arm64.txt +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/resources/demo.gif +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/resources/demo.tape +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/resources/ingestr.svg +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/AMPM.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Acronyms.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Colons.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Contractions.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/DateFormat.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ellipses.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/EmDash.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Exclamation.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/FirstPerson.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Gender.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/GenderBias.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/HeadingPunctuation.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Headings.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Latin.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/LyHyphens.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/OptionalPlurals.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ordinal.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/OxfordComma.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Parens.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Passive.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Periods.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Quotes.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Ranges.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Semicolons.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Slang.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Spacing.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Spelling.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Units.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/We.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/Will.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/WordList.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/meta.json +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/Google/vocab.txt +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/bruin/Ingestr.yml +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/styles/config/vocabularies/bruin/accept.txt +0 -0
- {ingestr-0.13.35 → ingestr-0.13.36}/test.env.template +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.36
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
Binary file
|
|
Binary file
|
|
@@ -41,7 +41,9 @@ ingestr ingest \
|
|
|
41
41
|
### **`--interval-end` (Optional)**
|
|
42
42
|
- **Description**: The end date for fetching historical exchange rates.
|
|
43
43
|
- **Value**: A date in the format `YYYY-MM-DD` (e.g., `'2025-03-28'`).
|
|
44
|
-
- **Purpose**: Defines the
|
|
44
|
+
- **Purpose**: Defines the end point for fetching historical data.
|
|
45
|
+
- If `--interval-start` is provided without `--interval-end`, `--interval-end` defaults to the current date and retrieves up until the latest published data.
|
|
46
|
+
- If `--interval-end` is provided without `--interval-start`, it will be ignored and the call will retrieve the last published data.
|
|
45
47
|
- For `latest` and `currencies` this parameter is ignored.
|
|
46
48
|
|
|
47
49
|
---
|
|
@@ -104,7 +106,7 @@ ingestr ingest \
|
|
|
104
106
|
- **Primary Key**: Composite key of `date` and `currency_name`.
|
|
105
107
|
- **Notes**:
|
|
106
108
|
- An optional start and end date can be added via the arguments `--interval-start` and optionally `--interval-end` to define the date range (see examples below). If no start date is specified, the date will default today's date (and thus return the latest exchange rates).
|
|
107
|
-
- If a start date but no end date is specified, then the end date will default to
|
|
109
|
+
- If a start date but no end date is specified, then the end date will default to today's date and ingestr will retrieve data up until the latest published data.
|
|
108
110
|
- Note that the [Frankfurter API](https://www.frankfurter.dev/) only publishes updates Monday-Friday. If the given date is on the weekend, the date will default to the previous Friday. The source is however implemented in ingestr in such a way as to avoid duplicating rows of data in this case (see [Incremental Loading - Replace](https://bruin-data.github.io/ingestr/getting-started/incremental-loading.html)).
|
|
109
111
|
|
|
110
112
|
#### **Example Table: Handling Weekend Dates**
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Freshdesk
|
|
2
|
+
|
|
3
|
+
[Freshdesk](https://www.freshworks.com/freshdesk/) is a cloud-based customer service platform that helps businesses manages customer support via multiple channels including email,phone,websites, and social media.
|
|
4
|
+
|
|
5
|
+
ingestr supports Freshdesk as a source.
|
|
6
|
+
|
|
7
|
+
## URI format
|
|
8
|
+
|
|
9
|
+
The URI format for Freshdesk is:
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
freshdesk://<domain>?api_key=<api_key>
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
URI parameters:
|
|
16
|
+
- `api_key` : The API token used for authentication with Freshdesk.
|
|
17
|
+
- `domain`: The domain of your Freshdesk account, found in your account URL. For example, if your account URL is https://my_company.freshdesk.com/, then `my_company` is your domain.
|
|
18
|
+
|
|
19
|
+
The URI is used to connect to the Freshdesk API for extracting data.
|
|
20
|
+
|
|
21
|
+
## Setting up a Freshdesk integration
|
|
22
|
+
|
|
23
|
+
Freshdesk requires a few steps to set up an integration, please follow dltHub [setup guide](https://dlthub.com/docs/dlt-ecosystem/verified-sources/freshdesk#setup-guide).
|
|
24
|
+
|
|
25
|
+
After completing the guide,, you will have your `api_key` and `domain`. For example, if your api_key is
|
|
26
|
+
`test_key` and domain `my_company`, you can use the following command to copy data from Freshdesk into a DuckDB database:
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
ingestr ingest --source-uri 'freshdesk://$DOMAIN?api_key=$API_KEY' --source-table 'contacts' --dest-uri duckdb:///freshdesk.duckdb --dest-table 'dest.contacts'
|
|
30
|
+
```
|
|
31
|
+
This command copies data from the Freshdesk source to DuckDB.
|
|
32
|
+
|
|
33
|
+
<img alt="freshdesk_img" src="../media/freshdesk_ingestion.png" />
|
|
34
|
+
|
|
35
|
+
## Available Tables
|
|
36
|
+
The Freshdesk source allows you to ingest the following tables:
|
|
37
|
+
|
|
38
|
+
| S.No. | Name | Description |
|
|
39
|
+
| ----- | --------- | ----------------------------------------------------------------------------------------- |
|
|
40
|
+
| 1. | agents | Users responsible for managing and resolving customer inquiries and support tickets. |
|
|
41
|
+
| 2. | companies | Customer organizations or groups that agents support. |
|
|
42
|
+
| 3. | contacts | Individuals or customers who reach out for support. |
|
|
43
|
+
| 4. | groups | Agents organized based on specific criteria. |
|
|
44
|
+
| 5. | roles | Predefined sets of permissions that determine what actions an agent can perform. |
|
|
45
|
+
| 6. | tickets | Customer inquiries or issues submitted via various channels like email, chat, phone, etc. |
|
|
46
|
+
|
|
47
|
+
Use these as the `--source-table` parameter in the `ingestr ingest` command.
|
|
@@ -19,31 +19,64 @@ URI parameters:
|
|
|
19
19
|
- `credentials_path`: The path to the service account JSON file.
|
|
20
20
|
- `property_id`: It is a unique number that identifies a particular property on Google Analytics. [Follow this guide](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) to know more about property ID.
|
|
21
21
|
|
|
22
|
-
## Setting up
|
|
22
|
+
## Setting up a Google Analytics Integration
|
|
23
23
|
Google Analytics requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/google_analytics#grab-google-service-account-credentials). Once you complete the guide, you should have an `.json` file and `project_id`.
|
|
24
24
|
|
|
25
|
-
##
|
|
26
|
-
Custom reports
|
|
25
|
+
## Available Tables:
|
|
26
|
+
### Custom reports
|
|
27
|
+
- `Custom reports`: allow you to retrieve data based on specific `dimensions` and `metrics`.
|
|
27
28
|
|
|
28
|
-
Custom Table Format:
|
|
29
|
+
#### Custom Table Format:
|
|
29
30
|
```
|
|
30
31
|
custom:<dimensions>:<metrics>
|
|
31
32
|
```
|
|
32
33
|
|
|
33
|
-
|
|
34
|
-
- `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/
|
|
35
|
-
- `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/
|
|
34
|
+
#### Parameters:
|
|
35
|
+
- `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#dimensions) to retrieve.
|
|
36
|
+
- `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#metrics) to retrieve.
|
|
36
37
|
|
|
37
|
-
|
|
38
|
+
#### Example
|
|
38
39
|
|
|
39
40
|
```sh
|
|
40
41
|
ingestr ingest \
|
|
41
42
|
--source-uri "googleanalytics://?credentials_path="ingestr/src/g_analytics.json&property_id=id123" \
|
|
42
|
-
--source-table "custom:
|
|
43
|
+
--source-table "custom:date:activeUsers" \
|
|
43
44
|
--dest-uri "duckdb:///analytics.duckdb" \
|
|
44
45
|
--dest-table "dest.custom"
|
|
45
46
|
```
|
|
46
47
|
|
|
47
48
|
This command will retrieve report and save it to the `dest.custom` table in the DuckDB database.
|
|
48
49
|
|
|
49
|
-
<img alt="google_analytics_img" src="../media/googleanalytics.png" />
|
|
50
|
+
<img alt="google_analytics_img" src="../media/googleanalytics.png" />
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
### Realtime reports
|
|
54
|
+
`Realtime reports`: allows you to retrieve data based on specific `dimensions`, `metrics`, with optional `minutes_ranges`.
|
|
55
|
+
|
|
56
|
+
#### Realtime Report Table Format:
|
|
57
|
+
```
|
|
58
|
+
realtime:<dimensions>:<metrics>
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
```
|
|
62
|
+
realtime:<dimensions>:<metrics>:<minutes_ranges>
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
#### Parameters:
|
|
66
|
+
- `dimensions`(required): A comma-separated list of [dimensions](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#dimensions) to retrieve.
|
|
67
|
+
- `metrics`(required): A comma-separated list of [metrics](https://developers.google.com/analytics/devguides/reporting/data/v1/exploration-api-schema#metrics) to retrieve.
|
|
68
|
+
- `minutes_ranges`(optional): Allows you to specify time windows for retrieving data. You can define up to two time ranges in your query, formatted as comma-separated values (e.g., "0-5,25-29"). Each range represents minutes in the past from the current time.
|
|
69
|
+
If no minute_ranges are specified, the system defaults to retrieving data from the last 30 minutes. For more information read [here](https://developers.google.com/analytics/devguides/reporting/data/v1/realtime-basics#minute_ranges)
|
|
70
|
+
|
|
71
|
+
#### Example
|
|
72
|
+
|
|
73
|
+
```sh
|
|
74
|
+
ingestr ingest \
|
|
75
|
+
--source-uri "googleanalytics://?credentials_path="ingestr/src/g_analytics.json&property_id=id123" \
|
|
76
|
+
--source-table "realtime:streamId:activeUsers:0-4,10-29" \
|
|
77
|
+
--dest-uri "duckdb:///analytics.duckdb" \
|
|
78
|
+
--dest-table "dest.realtime"
|
|
79
|
+
```
|
|
80
|
+
This command will retrieve report and save it to the `dest.realtime` table in the DuckDB database.
|
|
81
|
+
|
|
82
|
+
<img alt="google_analytics_realtime_report_img" src="../media/google_analytics_realtime_report.png"/>
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "v0.13.36"
|
|
@@ -53,6 +53,7 @@ from ingestr.src.sources import (
|
|
|
53
53
|
StripeAnalyticsSource,
|
|
54
54
|
TikTokSource,
|
|
55
55
|
ZendeskSource,
|
|
56
|
+
FreshdeskSource,
|
|
56
57
|
)
|
|
57
58
|
|
|
58
59
|
SQL_SOURCE_SCHEMES = [
|
|
@@ -148,6 +149,7 @@ class SourceDestinationFactory:
|
|
|
148
149
|
"kinesis": KinesisSource,
|
|
149
150
|
"pipedrive": PipedriveSource,
|
|
150
151
|
"frankfurter": FrankfurterSource,
|
|
152
|
+
"freshdesk": FreshdeskSource,
|
|
151
153
|
}
|
|
152
154
|
destinations: Dict[str, Type[DestinationProtocol]] = {
|
|
153
155
|
"bigquery": BigQueryDestination,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Any, Iterator
|
|
1
|
+
from typing import Any, Iterator
|
|
2
2
|
|
|
3
3
|
import dlt
|
|
4
4
|
from dlt.common.pendulum import pendulum
|
|
@@ -13,25 +13,28 @@ from ingestr.src.frankfurter.helpers import get_path_with_retry
|
|
|
13
13
|
max_table_nesting=0,
|
|
14
14
|
)
|
|
15
15
|
def frankfurter_source(
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
end_date: Optional[TAnyDateTime] = None,
|
|
16
|
+
start_date: TAnyDateTime,
|
|
17
|
+
end_date: TAnyDateTime,
|
|
19
18
|
) -> Any:
|
|
20
19
|
"""
|
|
21
20
|
A dlt source for the frankfurter.dev API. It groups several resources (in this case frankfurter.dev API endpoints) containing
|
|
22
21
|
various types of data: currencies, latest rates, historical rates.
|
|
23
|
-
|
|
24
|
-
Returns the appropriate resource based on the provided parameters.
|
|
25
22
|
"""
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
23
|
+
date_time = dlt.sources.incremental(
|
|
24
|
+
|
|
25
|
+
"date",
|
|
26
|
+
initial_value=start_date,
|
|
27
|
+
end_value=end_date,
|
|
28
|
+
range_start="closed",
|
|
29
|
+
range_end="closed",
|
|
30
|
+
)
|
|
29
31
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
return (
|
|
33
|
+
currencies(),
|
|
34
|
+
latest(),
|
|
35
|
+
exchange_rates(start_date=date_time, end_date=end_date),
|
|
32
36
|
|
|
33
|
-
|
|
34
|
-
return exchange_rates(start_date=start_date, end_date=end_date)
|
|
37
|
+
)
|
|
35
38
|
|
|
36
39
|
|
|
37
40
|
@dlt.resource(
|
|
@@ -53,13 +56,13 @@ def currencies() -> Iterator[dict]:
|
|
|
53
56
|
|
|
54
57
|
|
|
55
58
|
@dlt.resource(
|
|
56
|
-
write_disposition="
|
|
59
|
+
write_disposition="merge",
|
|
57
60
|
columns={
|
|
58
61
|
"date": {"data_type": "text"},
|
|
59
|
-
"
|
|
62
|
+
"currency_code": {"data_type": "text"},
|
|
60
63
|
"rate": {"data_type": "double"},
|
|
61
64
|
},
|
|
62
|
-
primary_key=["date", "
|
|
65
|
+
primary_key=["date", "currency_code"], # Composite primary key
|
|
63
66
|
)
|
|
64
67
|
def latest() -> Iterator[dict]:
|
|
65
68
|
"""
|
|
@@ -69,50 +72,54 @@ def latest() -> Iterator[dict]:
|
|
|
69
72
|
url = "latest?"
|
|
70
73
|
|
|
71
74
|
# Fetch data
|
|
72
|
-
|
|
75
|
+
data = get_path_with_retry(url)
|
|
73
76
|
|
|
74
77
|
# Extract rates and base currency
|
|
75
|
-
rates =
|
|
78
|
+
rates = data["rates"]
|
|
76
79
|
|
|
77
|
-
|
|
78
|
-
date = pendulum.now().to_date_string()
|
|
80
|
+
date = pendulum.parse(data["date"])
|
|
79
81
|
|
|
80
82
|
# Add the base currency (EUR) with a rate of 1.0
|
|
81
83
|
yield {
|
|
82
84
|
"date": date,
|
|
83
|
-
"
|
|
85
|
+
"currency_code": "EUR",
|
|
84
86
|
"rate": 1.0,
|
|
85
87
|
}
|
|
86
88
|
|
|
87
89
|
# Add all currencies and their rates
|
|
88
|
-
for
|
|
90
|
+
for currency_code, rate in rates.items():
|
|
89
91
|
yield {
|
|
90
92
|
"date": date,
|
|
91
|
-
"
|
|
93
|
+
"currency_code": currency_code,
|
|
92
94
|
"rate": rate,
|
|
93
95
|
}
|
|
94
96
|
|
|
95
97
|
|
|
96
98
|
@dlt.resource(
|
|
97
|
-
write_disposition="
|
|
99
|
+
write_disposition="merge",
|
|
98
100
|
columns={
|
|
99
101
|
"date": {"data_type": "text"},
|
|
100
|
-
"
|
|
102
|
+
"currency_code": {"data_type": "text"},
|
|
101
103
|
"rate": {"data_type": "double"},
|
|
102
104
|
},
|
|
103
|
-
primary_key=
|
|
105
|
+
primary_key=("date", "currency_code"), # Composite primary key
|
|
104
106
|
)
|
|
105
107
|
def exchange_rates(
|
|
106
|
-
start_date: TAnyDateTime,
|
|
107
108
|
end_date: TAnyDateTime,
|
|
109
|
+
start_date: dlt.sources.incremental[TAnyDateTime] = dlt.sources.incremental("date"),
|
|
108
110
|
) -> Iterator[dict]:
|
|
109
111
|
"""
|
|
110
112
|
Fetches exchange rates for a specified date range.
|
|
111
|
-
If only start_date is provided, fetches data
|
|
113
|
+
If only start_date is provided, fetches data until now.
|
|
112
114
|
If both start_date and end_date are provided, fetches data for each day in the range.
|
|
113
115
|
"""
|
|
114
|
-
|
|
115
|
-
|
|
116
|
+
# Ensure start_date.last_value is a pendulum.DateTime object
|
|
117
|
+
start_date_obj = ensure_pendulum_datetime(start_date.last_value) # type: ignore
|
|
118
|
+
start_date_str = start_date_obj.format("YYYY-MM-DD")
|
|
119
|
+
|
|
120
|
+
# Ensure end_date is a pendulum.DateTime object
|
|
121
|
+
end_date_obj = ensure_pendulum_datetime(end_date)
|
|
122
|
+
end_date_str = end_date_obj.format("YYYY-MM-DD")
|
|
116
123
|
|
|
117
124
|
# Compose the URL
|
|
118
125
|
url = f"{start_date_str}..{end_date_str}?"
|
|
@@ -121,22 +128,23 @@ def exchange_rates(
|
|
|
121
128
|
data = get_path_with_retry(url)
|
|
122
129
|
|
|
123
130
|
# Extract base currency and rates from the API response
|
|
124
|
-
base_currency = data["base"]
|
|
125
131
|
rates = data["rates"]
|
|
126
132
|
|
|
127
133
|
# Iterate over the rates dictionary (one entry per date)
|
|
128
134
|
for date, daily_rates in rates.items():
|
|
135
|
+
formatted_date = pendulum.parse(date)
|
|
136
|
+
|
|
129
137
|
# Add the base currency with a rate of 1.0
|
|
130
138
|
yield {
|
|
131
|
-
"date":
|
|
132
|
-
"
|
|
139
|
+
"date": formatted_date,
|
|
140
|
+
"currency_code": "EUR",
|
|
133
141
|
"rate": 1.0,
|
|
134
142
|
}
|
|
135
143
|
|
|
136
144
|
# Add all other currencies and their rates
|
|
137
|
-
for
|
|
145
|
+
for currency_code, rate in daily_rates.items():
|
|
138
146
|
yield {
|
|
139
|
-
"date":
|
|
140
|
-
"
|
|
147
|
+
"date": formatted_date,
|
|
148
|
+
"currency_code": currency_code,
|
|
141
149
|
"rate": rate,
|
|
142
150
|
}
|
|
@@ -8,7 +8,7 @@ FRANKFURTER_API_URL = "https://api.frankfurter.dev/v1/"
|
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def get_url_with_retry(url: str) -> StrAny:
|
|
11
|
-
r = requests.get(url)
|
|
11
|
+
r = requests.get(url, timeout=5)
|
|
12
12
|
return r.json() # type: ignore
|
|
13
13
|
|
|
14
14
|
|
|
@@ -19,7 +19,7 @@ def get_path_with_retry(path: str) -> StrAny:
|
|
|
19
19
|
def validate_dates(start_date: datetime, end_date: datetime) -> None:
|
|
20
20
|
current_date = pendulum.now()
|
|
21
21
|
|
|
22
|
-
# Check if start_date is in the
|
|
22
|
+
# Check if start_date is in the futurep
|
|
23
23
|
if start_date > current_date:
|
|
24
24
|
raise ValueError("Interval-start cannot be in the future.")
|
|
25
25
|
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""This source uses Freshdesk API and dlt to load data such as Agents, Companies, Tickets
|
|
2
|
+
etc. to the database"""
|
|
3
|
+
|
|
4
|
+
from typing import Any, Dict, Generator, Iterable, List, Optional
|
|
5
|
+
|
|
6
|
+
import dlt
|
|
7
|
+
from dlt.sources import DltResource
|
|
8
|
+
|
|
9
|
+
from .freshdesk_client import FreshdeskClient
|
|
10
|
+
from .settings import DEFAULT_ENDPOINTS
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dlt.source()
|
|
14
|
+
def freshdesk_source(
|
|
15
|
+
endpoints: Optional[List[str]] = None,
|
|
16
|
+
per_page: int = 100,
|
|
17
|
+
domain: str = dlt.secrets.value,
|
|
18
|
+
api_secret_key: str = dlt.secrets.value,
|
|
19
|
+
) -> Iterable[DltResource]:
|
|
20
|
+
"""
|
|
21
|
+
Retrieves data from specified Freshdesk API endpoints.
|
|
22
|
+
|
|
23
|
+
This source supports pagination and incremental data loading. It fetches data from a list of
|
|
24
|
+
specified endpoints, or defaults to predefined endpoints in 'settings.py'.
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
endpoints: A list of Freshdesk API endpoints to fetch. Deafults to 'settings.py'.
|
|
28
|
+
per_page: The number of items to fetch per page, with a maximum of 100.
|
|
29
|
+
domain: The Freshdesk domain from which to fetch the data. Defaults to 'config.toml'.
|
|
30
|
+
api_secret_key: Freshdesk API key. Defaults to 'secrets.toml'.
|
|
31
|
+
|
|
32
|
+
Yields:
|
|
33
|
+
Iterable[DltResource]: Resources with data updated after the last 'updated_at'
|
|
34
|
+
timestamp for each endpoint.
|
|
35
|
+
"""
|
|
36
|
+
# Instantiate FreshdeskClient with the provided domain and API key
|
|
37
|
+
freshdesk = FreshdeskClient(api_key=api_secret_key, domain=domain)
|
|
38
|
+
|
|
39
|
+
def incremental_resource(
|
|
40
|
+
endpoint: str,
|
|
41
|
+
updated_at: Optional[Any] = dlt.sources.incremental(
|
|
42
|
+
"updated_at", initial_value="2022-01-01T00:00:00Z"
|
|
43
|
+
),
|
|
44
|
+
) -> Generator[Dict[Any, Any], Any, None]:
|
|
45
|
+
"""
|
|
46
|
+
Fetches and yields paginated data from a specified API endpoint.
|
|
47
|
+
Each page of data is fetched based on the `updated_at` timestamp
|
|
48
|
+
to ensure incremental loading.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
# Retrieve the last updated timestamp to fetch only new or updated records.
|
|
52
|
+
if updated_at is not None:
|
|
53
|
+
updated_at = updated_at.last_value
|
|
54
|
+
|
|
55
|
+
# Use the FreshdeskClient instance to fetch paginated responses
|
|
56
|
+
yield from freshdesk.paginated_response(
|
|
57
|
+
endpoint=endpoint,
|
|
58
|
+
per_page=per_page,
|
|
59
|
+
updated_at=updated_at,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
# Set default endpoints if not provided
|
|
63
|
+
endpoints = endpoints or DEFAULT_ENDPOINTS
|
|
64
|
+
|
|
65
|
+
# For each endpoint, create and yield a DLT resource
|
|
66
|
+
for endpoint in endpoints:
|
|
67
|
+
yield dlt.resource(
|
|
68
|
+
incremental_resource,
|
|
69
|
+
name=endpoint,
|
|
70
|
+
write_disposition="merge",
|
|
71
|
+
primary_key="id",
|
|
72
|
+
)(endpoint=endpoint)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Freshdesk Client for making authenticated requests"""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import time
|
|
5
|
+
from typing import Any, Dict, Iterable, Optional
|
|
6
|
+
|
|
7
|
+
from dlt.common.typing import TDataItem
|
|
8
|
+
from dlt.sources.helpers import requests
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class FreshdeskClient:
|
|
12
|
+
"""
|
|
13
|
+
Client for making authenticated requests to the Freshdesk API. It incorporates API requests with
|
|
14
|
+
rate limit and pagination.
|
|
15
|
+
|
|
16
|
+
Attributes:
|
|
17
|
+
api_key (str): The API key used for authenticating requests to the Freshdesk API.
|
|
18
|
+
domain (str): The Freshdesk domain specific to the user, used in constructing the base URL.
|
|
19
|
+
base_url (str): The base URL constructed from the domain, targeting the Freshdesk API v2.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, api_key: str, domain: str):
|
|
23
|
+
# Initialize the FreshdeskClient instance with API key and domain.
|
|
24
|
+
# The API key is used for authentication with the Freshdesk API.
|
|
25
|
+
# The domain specifies the unique Freshdesk domain of the user.
|
|
26
|
+
|
|
27
|
+
# Store the API key provided during initialization.
|
|
28
|
+
self.api_key = api_key
|
|
29
|
+
# Store the Freshdesk domain provided during initialization.
|
|
30
|
+
self.domain = domain
|
|
31
|
+
|
|
32
|
+
# Construct the base URL for the API requests.
|
|
33
|
+
# This URL is formed by appending the domain to the standard Freshdesk API base URL format.
|
|
34
|
+
# All API requests will use this base URL as their starting point.
|
|
35
|
+
self.base_url = f"https://{domain}.freshdesk.com/api/v2"
|
|
36
|
+
|
|
37
|
+
def _request_with_rate_limit(self, url: str, **kwargs: Any) -> requests.Response:
|
|
38
|
+
"""
|
|
39
|
+
Handles rate limits in HTTP requests and ensures
|
|
40
|
+
that the client doesn't exceed the limit set by the server.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
while True:
|
|
44
|
+
try:
|
|
45
|
+
response = requests.get(url, **kwargs, auth=(self.api_key, "X"))
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
|
|
48
|
+
return response
|
|
49
|
+
except requests.HTTPError as e:
|
|
50
|
+
if e.response.status_code == 429:
|
|
51
|
+
# Get the 'Retry-After' header to know how long to wait
|
|
52
|
+
# Fallback to 60 seconds if header is missing
|
|
53
|
+
seconds_to_wait = int(e.response.headers.get("Retry-After", 60))
|
|
54
|
+
# Log a warning message
|
|
55
|
+
logging.warning(
|
|
56
|
+
"Rate limited. Waiting to retry after: %s secs", seconds_to_wait
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Wait for the specified number of seconds before retrying
|
|
60
|
+
time.sleep(seconds_to_wait)
|
|
61
|
+
else:
|
|
62
|
+
# If the error is not a rate limit (429), raise the exception to be
|
|
63
|
+
# handled elsewhere or stop execution
|
|
64
|
+
raise
|
|
65
|
+
|
|
66
|
+
def paginated_response(
|
|
67
|
+
self,
|
|
68
|
+
endpoint: str,
|
|
69
|
+
per_page: int,
|
|
70
|
+
updated_at: Optional[str] = None,
|
|
71
|
+
) -> Iterable[TDataItem]:
|
|
72
|
+
"""
|
|
73
|
+
Fetches a paginated response from a specified endpoint.
|
|
74
|
+
|
|
75
|
+
This method will continuously fetch data from the given endpoint,
|
|
76
|
+
page by page, until no more data is available or until it reaches data
|
|
77
|
+
updated at the specified timestamp.
|
|
78
|
+
"""
|
|
79
|
+
page = 1
|
|
80
|
+
while True:
|
|
81
|
+
# Construct the URL for the specific endpoint
|
|
82
|
+
url = f"{self.base_url}/{endpoint}"
|
|
83
|
+
|
|
84
|
+
params: Dict[str, Any] = {"per_page": per_page, "page": page}
|
|
85
|
+
|
|
86
|
+
# Implement date range splitting logic here, if applicable
|
|
87
|
+
if endpoint in ["tickets", "contacts"]:
|
|
88
|
+
param_key = (
|
|
89
|
+
"updated_since" if endpoint == "tickets" else "_updated_since"
|
|
90
|
+
)
|
|
91
|
+
if updated_at:
|
|
92
|
+
params[param_key] = updated_at
|
|
93
|
+
|
|
94
|
+
# Handle requests with rate-limiting
|
|
95
|
+
# A maximum of 300 pages (30000 tickets) will be returned.
|
|
96
|
+
response = self._request_with_rate_limit(url, params=params)
|
|
97
|
+
data = response.json()
|
|
98
|
+
|
|
99
|
+
if not data:
|
|
100
|
+
break # Stop if no data or max page limit reached
|
|
101
|
+
yield data
|
|
102
|
+
page += 1
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""
|
|
2
|
+
This module defines default settings for the Freshdesk integration.
|
|
3
|
+
|
|
4
|
+
It specifies a list of default endpoints to be used when interacting with the Freshdesk API,
|
|
5
|
+
covering common entities such as agents, companies, contacts, groups, roles, and tickets.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Define default endpoints for the Freshdesk API integration.
|
|
9
|
+
DEFAULT_ENDPOINTS = ["agents", "companies", "contacts", "groups", "roles", "tickets"]
|
|
@@ -13,9 +13,10 @@ from google.analytics.data_v1beta import BetaAnalyticsDataClient
|
|
|
13
13
|
from google.analytics.data_v1beta.types import (
|
|
14
14
|
Dimension,
|
|
15
15
|
Metric,
|
|
16
|
+
MinuteRange,
|
|
16
17
|
)
|
|
17
18
|
|
|
18
|
-
from .helpers import get_report
|
|
19
|
+
from .helpers import get_realtime_report, get_report
|
|
19
20
|
|
|
20
21
|
|
|
21
22
|
@dlt.source(max_table_nesting=0)
|
|
@@ -29,6 +30,7 @@ def google_analytics(
|
|
|
29
30
|
start_date: Optional[pendulum.DateTime] = pendulum.datetime(2024, 1, 1),
|
|
30
31
|
end_date: Optional[pendulum.DateTime] = None,
|
|
31
32
|
rows_per_page: int = 10000,
|
|
33
|
+
minute_range_objects: List[MinuteRange] | None = None,
|
|
32
34
|
) -> List[DltResource]:
|
|
33
35
|
try:
|
|
34
36
|
property_id = int(property_id)
|
|
@@ -58,7 +60,7 @@ def google_analytics(
|
|
|
58
60
|
dimensions = query["dimensions"]
|
|
59
61
|
|
|
60
62
|
@dlt.resource(
|
|
61
|
-
name="
|
|
63
|
+
name="custom",
|
|
62
64
|
merge_key=datetime_dimension,
|
|
63
65
|
write_disposition="merge",
|
|
64
66
|
)
|
|
@@ -87,6 +89,22 @@ def google_analytics(
|
|
|
87
89
|
end_date=end_date,
|
|
88
90
|
)
|
|
89
91
|
|
|
92
|
+
# real time report
|
|
93
|
+
@dlt.resource(
|
|
94
|
+
name="realtime",
|
|
95
|
+
merge_key="ingested_at",
|
|
96
|
+
write_disposition="merge",
|
|
97
|
+
)
|
|
98
|
+
def real_time_report() -> Iterator[TDataItem]:
|
|
99
|
+
yield from get_realtime_report(
|
|
100
|
+
client=client,
|
|
101
|
+
property_id=property_id,
|
|
102
|
+
dimension_list=[Dimension(name=dimension) for dimension in dimensions],
|
|
103
|
+
metric_list=[Metric(name=metric) for metric in query["metrics"]],
|
|
104
|
+
per_page=rows_per_page,
|
|
105
|
+
minute_range_objects=minute_range_objects,
|
|
106
|
+
)
|
|
107
|
+
|
|
90
108
|
# res = dlt.resource(
|
|
91
109
|
# basic_report, name="basic_report", merge_key=datetime_dimension, write_disposition="merge"
|
|
92
110
|
# )(
|
|
@@ -103,4 +121,4 @@ def google_analytics(
|
|
|
103
121
|
# ),
|
|
104
122
|
# )
|
|
105
123
|
|
|
106
|
-
return [basic_report]
|
|
124
|
+
return [basic_report, real_time_report]
|