ingestr 0.13.10__tar.gz → 0.13.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- {ingestr-0.13.10 → ingestr-0.13.11}/PKG-INFO +17 -11
- {ingestr-0.13.10 → ingestr-0.13.11}/README.md +5 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/config.mjs +1 -0
- ingestr-0.13.11/docs/supported-sources/salesforce.md +68 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/applovin/__init__.py +8 -7
- ingestr-0.13.11/ingestr/src/buildinfo.py +1 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/factory.py +2 -0
- ingestr-0.13.11/ingestr/src/salesforce/__init__.py +149 -0
- ingestr-0.13.11/ingestr/src/salesforce/helpers.py +64 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sources.py +71 -6
- {ingestr-0.13.10 → ingestr-0.13.11}/requirements.txt +11 -12
- ingestr-0.13.10/ingestr/src/buildinfo.py +0 -1
- {ingestr-0.13.10 → ingestr-0.13.11}/.dockerignore +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.githooks/pre-commit-hook.sh +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/deploy-docs.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/release.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/secrets-scan.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.github/workflows/tests.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.gitignore +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.gitleaksignore +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.python-version +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/.vale.ini +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/Dockerfile +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/LICENSE.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/Makefile +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/theme/custom.css +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/.vitepress/theme/index.js +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/commands/example-uris.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/commands/ingest.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/core-concepts.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/incremental-loading.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/quickstart.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/getting-started/telemetry.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/index.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/applovin_max.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/athena.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/clickhouse_img.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/github.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/googleanalytics.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/linkedin_ads.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/personio.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/media/tiktok.png +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/adjust.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/airtable.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/applovin.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/applovin_max.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/appsflyer.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/appstore.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/asana.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/athena.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/bigquery.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/chess.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/clickhouse.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/csv.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/custom_queries.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/databricks.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/duckdb.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/dynamodb.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/facebook-ads.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gcs.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/github.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/google-ads.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/google_analytics.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gorgias.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/gsheets.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/hubspot.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/kafka.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/klaviyo.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/linkedin_ads.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mongodb.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mssql.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/mysql.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/notion.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/oracle.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/personio.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/postgres.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/redshift.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/s3.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/sap-hana.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/shopify.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/slack.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/snowflake.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/sqlite.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/stripe.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/tiktok-ads.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/docs/supported-sources/zendesk.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/main.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/.gitignore +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/adjust/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/adjust/adjust_helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/airtable/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/applovin_max/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appsflyer/_init_.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appsflyer/client.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/client.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/errors.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/models.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/appstore/resources.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/arrow/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/asana_source/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/blob.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/chess/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/destinations.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/dynamodb/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/errors.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/exceptions.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/facebook_ads/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filesystem/readers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/filters.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/queries.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/github/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/field.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/metrics.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/predicates.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_ads/reports.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_analytics/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_analytics/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/README.md +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/api_calls.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/google_sheets/helpers/data_processing.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/gorgias/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/gorgias/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/hubspot/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/kafka/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/kafka/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/_init_.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/client.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/klaviyo/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/dimension_time_enum.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/linkedin_ads/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/loader.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/mongodb/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/mongodb/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/client.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/helpers/database.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/notion/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/personio/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/personio/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/exceptions.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/shopify/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/slack/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sql_database/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/sql_database/callbacks.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/stripe_analytics/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/table_definition.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/telemetry/event.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/testdata/fakebqcredentials.json +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/tiktok_ads/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/tiktok_ads/tiktok_helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/time.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/version.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/__init__.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/api_helpers.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/credentials.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/helpers/talk_api.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/src/zendesk/settings.py +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/.gitignore +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/create_replace.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_expected.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_part1.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/delete_insert_part2.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_expected.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_part1.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/ingestr/testdata/merge_part2.csv +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/package-lock.json +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/package.json +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/pyproject.toml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/requirements-dev.txt +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/resources/demo.gif +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/resources/demo.tape +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/resources/ingestr.svg +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/AMPM.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Acronyms.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Colons.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Contractions.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/DateFormat.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ellipses.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/EmDash.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Exclamation.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/FirstPerson.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Gender.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/GenderBias.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/HeadingPunctuation.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Headings.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Latin.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/LyHyphens.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/OptionalPlurals.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ordinal.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/OxfordComma.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Parens.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Passive.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Periods.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Quotes.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Ranges.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Semicolons.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Slang.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Spacing.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Spelling.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Units.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/We.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/Will.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/WordList.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/meta.json +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/Google/vocab.txt +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/bruin/Ingestr.yml +0 -0
- {ingestr-0.13.10 → ingestr-0.13.11}/styles/config/vocabularies/bruin/accept.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.11
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -18,42 +18,43 @@ Requires-Dist: asana==3.2.3
|
|
|
18
18
|
Requires-Dist: clickhouse-connect==0.8.14
|
|
19
19
|
Requires-Dist: clickhouse-driver==0.2.9
|
|
20
20
|
Requires-Dist: clickhouse-sqlalchemy==0.2.7
|
|
21
|
-
Requires-Dist: confluent-kafka>=2.
|
|
21
|
+
Requires-Dist: confluent-kafka>=2.8.0
|
|
22
22
|
Requires-Dist: databricks-sql-connector==2.9.3
|
|
23
23
|
Requires-Dist: dataclasses-json==0.6.7
|
|
24
|
-
Requires-Dist: dlt==1.
|
|
25
|
-
Requires-Dist: duckdb-engine==0.
|
|
26
|
-
Requires-Dist: duckdb==1.
|
|
24
|
+
Requires-Dist: dlt==1.6.1
|
|
25
|
+
Requires-Dist: duckdb-engine==0.15.0
|
|
26
|
+
Requires-Dist: duckdb==1.2.0
|
|
27
27
|
Requires-Dist: facebook-business==20.0.0
|
|
28
28
|
Requires-Dist: flatten-json==0.1.14
|
|
29
29
|
Requires-Dist: gcsfs==2024.10.0
|
|
30
30
|
Requires-Dist: google-ads==25.1.0
|
|
31
|
-
Requires-Dist: google-analytics-data==0.18.
|
|
31
|
+
Requires-Dist: google-analytics-data==0.18.17
|
|
32
32
|
Requires-Dist: google-api-python-client==2.130.0
|
|
33
33
|
Requires-Dist: google-cloud-bigquery-storage==2.24.0
|
|
34
|
-
Requires-Dist: mysql-connector-python==9.
|
|
34
|
+
Requires-Dist: mysql-connector-python==9.2.0
|
|
35
35
|
Requires-Dist: pendulum==3.0.0
|
|
36
36
|
Requires-Dist: psutil==6.1.1
|
|
37
37
|
Requires-Dist: psycopg2-binary==2.9.10
|
|
38
38
|
Requires-Dist: py-machineid==0.6.0
|
|
39
39
|
Requires-Dist: pyairtable==2.3.3
|
|
40
40
|
Requires-Dist: pyarrow==18.1.0
|
|
41
|
-
Requires-Dist: pyathena==3.
|
|
42
|
-
Requires-Dist: pymongo==4.
|
|
41
|
+
Requires-Dist: pyathena==3.12.2
|
|
42
|
+
Requires-Dist: pymongo==4.11.1
|
|
43
43
|
Requires-Dist: pymysql==1.1.1
|
|
44
44
|
Requires-Dist: pyrate-limiter==3.7.0
|
|
45
45
|
Requires-Dist: redshift-connector==2.1.5
|
|
46
46
|
Requires-Dist: rich==13.9.4
|
|
47
47
|
Requires-Dist: rudder-sdk-python==2.1.4
|
|
48
48
|
Requires-Dist: s3fs==2024.10.0
|
|
49
|
+
Requires-Dist: simple-salesforce==1.12.6
|
|
49
50
|
Requires-Dist: snowflake-sqlalchemy==1.6.1
|
|
50
|
-
Requires-Dist: sqlalchemy-bigquery==1.12.
|
|
51
|
+
Requires-Dist: sqlalchemy-bigquery==1.12.1
|
|
51
52
|
Requires-Dist: sqlalchemy-hana==2.0.0
|
|
52
53
|
Requires-Dist: sqlalchemy-redshift==0.8.14
|
|
53
54
|
Requires-Dist: sqlalchemy2-stubs==0.0.2a38
|
|
54
55
|
Requires-Dist: sqlalchemy==1.4.52
|
|
55
56
|
Requires-Dist: stripe==10.7.0
|
|
56
|
-
Requires-Dist: tqdm==4.67.
|
|
57
|
+
Requires-Dist: tqdm==4.67.1
|
|
57
58
|
Requires-Dist: typer==0.13.1
|
|
58
59
|
Requires-Dist: types-requests==2.32.0.20240907
|
|
59
60
|
Provides-Extra: odbc
|
|
@@ -312,6 +313,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
312
313
|
<td>✅</td>
|
|
313
314
|
<td>-</td>
|
|
314
315
|
</tr>
|
|
316
|
+
<tr>
|
|
317
|
+
<td>Salesforce</td>
|
|
318
|
+
<td>✅</td>
|
|
319
|
+
<td>-</td>
|
|
320
|
+
</tr>
|
|
315
321
|
<tr>
|
|
316
322
|
<td>Shopify</td>
|
|
317
323
|
<td>✅</td>
|
|
@@ -117,6 +117,7 @@ export default defineConfig({
|
|
|
117
117
|
{ text: "Notion", link: "/supported-sources/notion.md" },
|
|
118
118
|
{ text: "Personio", link: "/supported-sources/personio.md" },
|
|
119
119
|
{ text: "S3", link: "/supported-sources/s3.md" },
|
|
120
|
+
{ text: "Salesforce", link: "/supported-sources/salesforce.md" },
|
|
120
121
|
{ text: "Shopify", link: "/supported-sources/shopify.md" },
|
|
121
122
|
{ text: "Slack", link: "/supported-sources/slack.md" },
|
|
122
123
|
{ text: "Stripe", link: "/supported-sources/stripe.md" },
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# Salesforce
|
|
2
|
+
[Salesforce](https://www.salesforce.com/) is a cloud-based customer relationship management (CRM) platform that helps businesses manage sales, customer interactions, and business processes. It provides tools for sales automation, customer service, marketing, analytics, and application development.
|
|
3
|
+
|
|
4
|
+
Ingestr supports Salesforce as a source.
|
|
5
|
+
|
|
6
|
+
## URI format
|
|
7
|
+
|
|
8
|
+
The URI format for Salesforce is as follows:
|
|
9
|
+
```
|
|
10
|
+
salesforce://?username=<username>&password=<password>&token=<token>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
URI parameters:
|
|
14
|
+
- `username` is your Salesforce account username.
|
|
15
|
+
- `password` is your Salesforce account password.
|
|
16
|
+
- `token` is your Salesforce security token.
|
|
17
|
+
|
|
18
|
+
You can obtain your security token by logging into your Salesforce account and navigating to the user settings under "Reset My Security Token."
|
|
19
|
+
|
|
20
|
+
## Setting up a Salesforce Integration
|
|
21
|
+
|
|
22
|
+
You can obtain an OAuth access token by setting up a connected app in Salesforce and using OAuth 2.0 authentication. For more information, see [Salesforce API Authentication](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/quickstart_oauth.htm).
|
|
23
|
+
|
|
24
|
+
## Example
|
|
25
|
+
|
|
26
|
+
Let's say:
|
|
27
|
+
* Your Salesforce username is `user`.
|
|
28
|
+
* Your password is `password123`.
|
|
29
|
+
* Your security token is `fake_token`.
|
|
30
|
+
* You want to ingest `account` data from your salesforce account
|
|
31
|
+
* You want to save this data in a duckdb database `sf.db` under the table `public.account`
|
|
32
|
+
|
|
33
|
+
You can run the following command to achieve this:
|
|
34
|
+
```sh
|
|
35
|
+
ingestr ingest \
|
|
36
|
+
--source-uri "salesforce://?username=user&password=password123&token=fake_token" \
|
|
37
|
+
--source-table "account" \
|
|
38
|
+
--dest-uri "duckdb:///sf.db" \
|
|
39
|
+
--dest-table "public.account"
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Tables
|
|
43
|
+
|
|
44
|
+
Salesforce source allows ingesting the following objects into separate tables:
|
|
45
|
+
|
|
46
|
+
| **Table** | **Mode** | **Description** |
|
|
47
|
+
|---------------------------|-----------|----------------|
|
|
48
|
+
| `user` | replace | Refers to an individual who has access to a Salesforce org or instance. |
|
|
49
|
+
| `user_role` | replace | A standard object that represents a role within the organization's hierarchy. |
|
|
50
|
+
| `opportunity` | merge | Represents a sales opportunity for a specific account or contact. |
|
|
51
|
+
| `opportunity_line_item` | merge | Represents individual line items or products associated with an Opportunity. |
|
|
52
|
+
| `opportunity_contact_role` | merge | Represents the association between an Opportunity and a Contact. |
|
|
53
|
+
| `account` | merge | Individual or organization that interacts with your business. |
|
|
54
|
+
| `contact` | replace | An individual person associated with an account or organization. |
|
|
55
|
+
| `lead` | replace | Prospective customer/individual/org. that has shown interest in a company's products/services. |
|
|
56
|
+
| `campaign` | replace | Marketing initiative or project designed to achieve specific goals, such as generating leads. |
|
|
57
|
+
| `campaign_member` | merge | Association between a Contact or Lead and a Campaign. |
|
|
58
|
+
| `product` | replace | For managing and organizing your product-related data within the Salesforce ecosystem. |
|
|
59
|
+
| `pricebook` | replace | Used to manage product pricing and create price books. |
|
|
60
|
+
| `pricebook_entry` | replace | Represents a specific price for a product in a price book. |
|
|
61
|
+
| `task` | merge | Used to track and manage various activities and tasks within the Salesforce platform. |
|
|
62
|
+
| `event` | merge | Used to track and manage calendar-based events, such as meetings, appointments, or calls. |
|
|
63
|
+
|
|
64
|
+
Use these as `--source-table` parameters in the `ingestr ingest` command.
|
|
65
|
+
|
|
66
|
+
> [!WARNING]
|
|
67
|
+
> Salesforce API limits may affect the frequency and volume of data ingestion. Incremental loading is supported for objects with the `SystemModstamp` field, but some objects may require full-refresh loads. This is indicated by `mode` in the tables above. Tables with mode `replace` don't support incremental loads, while the ones with `merge` do.
|
|
68
|
+
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
from datetime import datetime,
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
2
|
from enum import Enum
|
|
3
3
|
from typing import Dict, List, Optional
|
|
4
|
-
from requests import Response
|
|
5
4
|
|
|
6
5
|
import dlt
|
|
7
6
|
from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
|
|
7
|
+
from requests import Response
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class InvalidCustomReportError(Exception):
|
|
@@ -13,9 +13,11 @@ class InvalidCustomReportError(Exception):
|
|
|
13
13
|
"Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
+
|
|
16
17
|
class ClientError(Exception):
|
|
17
18
|
pass
|
|
18
19
|
|
|
20
|
+
|
|
19
21
|
TYPE_HINTS = {
|
|
20
22
|
"application_is_hidden": {"data_type": "bool"},
|
|
21
23
|
"average_cpa": {"data_type": "double"},
|
|
@@ -119,7 +121,6 @@ def applovin_source(
|
|
|
119
121
|
end_date: Optional[str],
|
|
120
122
|
custom: Optional[str],
|
|
121
123
|
):
|
|
122
|
-
|
|
123
124
|
backfill = False
|
|
124
125
|
if end_date is None:
|
|
125
126
|
backfill = True
|
|
@@ -127,7 +128,7 @@ def applovin_source(
|
|
|
127
128
|
# use the greatest of yesterday and start_date
|
|
128
129
|
end_date = max(
|
|
129
130
|
datetime.now(timezone.utc) - timedelta(days=1),
|
|
130
|
-
datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
|
|
131
|
+
datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc),
|
|
131
132
|
).strftime("%Y-%m-%d")
|
|
132
133
|
|
|
133
134
|
config: RESTAPIConfig = {
|
|
@@ -157,7 +158,7 @@ def applovin_source(
|
|
|
157
158
|
"paginator": "single_page",
|
|
158
159
|
"response_actions": [
|
|
159
160
|
http_error_handler,
|
|
160
|
-
]
|
|
161
|
+
],
|
|
161
162
|
},
|
|
162
163
|
},
|
|
163
164
|
"resources": [
|
|
@@ -177,8 +178,7 @@ def applovin_source(
|
|
|
177
178
|
"advertiser-probabilistic-report",
|
|
178
179
|
"probabilisticReport",
|
|
179
180
|
exclude(
|
|
180
|
-
REPORT_SCHEMA[ReportType.ADVERTISER],
|
|
181
|
-
PROBABILISTIC_REPORT_EXCLUDE
|
|
181
|
+
REPORT_SCHEMA[ReportType.ADVERTISER], PROBABILISTIC_REPORT_EXCLUDE
|
|
182
182
|
),
|
|
183
183
|
ReportType.ADVERTISER,
|
|
184
184
|
),
|
|
@@ -256,6 +256,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
|
|
|
256
256
|
def build_type_hints(cols: List[str]) -> dict:
|
|
257
257
|
return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
|
|
258
258
|
|
|
259
|
+
|
|
259
260
|
def http_error_handler(resp: Response):
|
|
260
261
|
if not resp.ok:
|
|
261
262
|
raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
version = "v0.13.11"
|
|
@@ -42,6 +42,7 @@ from ingestr.src.sources import (
|
|
|
42
42
|
MongoDbSource,
|
|
43
43
|
NotionSource,
|
|
44
44
|
S3Source,
|
|
45
|
+
SalesforceSource,
|
|
45
46
|
ShopifySource,
|
|
46
47
|
SlackSource,
|
|
47
48
|
SqlSource,
|
|
@@ -137,6 +138,7 @@ class SourceDestinationFactory:
|
|
|
137
138
|
"linkedinads": LinkedInAdsSource,
|
|
138
139
|
"applovin": AppLovinSource,
|
|
139
140
|
"applovinmax": ApplovinMaxSource,
|
|
141
|
+
"salesforce": SalesforceSource,
|
|
140
142
|
"personio": PersonioSource,
|
|
141
143
|
}
|
|
142
144
|
destinations: Dict[str, Type[DestinationProtocol]] = {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from typing import Iterable
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
from dlt.common.typing import TDataItem
|
|
5
|
+
from dlt.sources import DltResource, incremental
|
|
6
|
+
from simple_salesforce import Salesforce
|
|
7
|
+
|
|
8
|
+
from .helpers import get_records
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dlt.source(name="salesforce")
|
|
12
|
+
def salesforce_source(
|
|
13
|
+
username: str,
|
|
14
|
+
password: str,
|
|
15
|
+
token: str,
|
|
16
|
+
) -> Iterable[DltResource]:
|
|
17
|
+
"""
|
|
18
|
+
Retrieves data from Salesforce using the Salesforce API.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
username (str): The username for authentication.
|
|
22
|
+
password (str): The password for authentication.
|
|
23
|
+
token (str): The security token for authentication.
|
|
24
|
+
|
|
25
|
+
Yields:
|
|
26
|
+
DltResource: Data resources from Salesforce.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
client = Salesforce(username, password, token)
|
|
30
|
+
|
|
31
|
+
# define resources
|
|
32
|
+
@dlt.resource(write_disposition="replace")
|
|
33
|
+
def user() -> Iterable[TDataItem]:
|
|
34
|
+
yield get_records(client, "User")
|
|
35
|
+
|
|
36
|
+
@dlt.resource(write_disposition="replace")
|
|
37
|
+
def user_role() -> Iterable[TDataItem]:
|
|
38
|
+
yield get_records(client, "UserRole")
|
|
39
|
+
|
|
40
|
+
@dlt.resource(write_disposition="merge")
|
|
41
|
+
def opportunity(
|
|
42
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
43
|
+
"SystemModstamp", initial_value=None
|
|
44
|
+
),
|
|
45
|
+
) -> Iterable[TDataItem]:
|
|
46
|
+
yield get_records(
|
|
47
|
+
client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@dlt.resource(write_disposition="merge")
|
|
51
|
+
def opportunity_line_item(
|
|
52
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
53
|
+
"SystemModstamp", initial_value=None
|
|
54
|
+
),
|
|
55
|
+
) -> Iterable[TDataItem]:
|
|
56
|
+
yield get_records(
|
|
57
|
+
client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
@dlt.resource(write_disposition="merge")
|
|
61
|
+
def opportunity_contact_role(
|
|
62
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
63
|
+
"SystemModstamp", initial_value=None
|
|
64
|
+
),
|
|
65
|
+
) -> Iterable[TDataItem]:
|
|
66
|
+
yield get_records(
|
|
67
|
+
client,
|
|
68
|
+
"OpportunityContactRole",
|
|
69
|
+
last_timestamp.last_value,
|
|
70
|
+
"SystemModstamp",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
@dlt.resource(write_disposition="merge")
|
|
74
|
+
def account(
|
|
75
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
76
|
+
"LastModifiedDate", initial_value=None
|
|
77
|
+
),
|
|
78
|
+
) -> Iterable[TDataItem]:
|
|
79
|
+
yield get_records(
|
|
80
|
+
client, "Account", last_timestamp.last_value, "LastModifiedDate"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@dlt.resource(write_disposition="replace")
|
|
84
|
+
def contact() -> Iterable[TDataItem]:
|
|
85
|
+
yield get_records(client, "Contact")
|
|
86
|
+
|
|
87
|
+
@dlt.resource(write_disposition="replace")
|
|
88
|
+
def lead() -> Iterable[TDataItem]:
|
|
89
|
+
yield get_records(client, "Lead")
|
|
90
|
+
|
|
91
|
+
@dlt.resource(write_disposition="replace")
|
|
92
|
+
def campaign() -> Iterable[TDataItem]:
|
|
93
|
+
yield get_records(client, "Campaign")
|
|
94
|
+
|
|
95
|
+
@dlt.resource(write_disposition="merge")
|
|
96
|
+
def campaign_member(
|
|
97
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
98
|
+
"SystemModstamp", initial_value=None
|
|
99
|
+
),
|
|
100
|
+
) -> Iterable[TDataItem]:
|
|
101
|
+
yield get_records(
|
|
102
|
+
client, "CampaignMember", last_timestamp.last_value, "SystemModstamp"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
@dlt.resource(write_disposition="replace")
|
|
106
|
+
def product() -> Iterable[TDataItem]:
|
|
107
|
+
yield get_records(client, "Product2")
|
|
108
|
+
|
|
109
|
+
@dlt.resource(write_disposition="replace")
|
|
110
|
+
def pricebook() -> Iterable[TDataItem]:
|
|
111
|
+
yield get_records(client, "Pricebook2")
|
|
112
|
+
|
|
113
|
+
@dlt.resource(write_disposition="replace")
|
|
114
|
+
def pricebook_entry() -> Iterable[TDataItem]:
|
|
115
|
+
yield get_records(client, "PricebookEntry")
|
|
116
|
+
|
|
117
|
+
@dlt.resource(write_disposition="merge")
|
|
118
|
+
def task(
|
|
119
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
120
|
+
"SystemModstamp", initial_value=None
|
|
121
|
+
),
|
|
122
|
+
) -> Iterable[TDataItem]:
|
|
123
|
+
yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
|
|
124
|
+
|
|
125
|
+
@dlt.resource(write_disposition="merge")
|
|
126
|
+
def event(
|
|
127
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
128
|
+
"SystemModstamp", initial_value=None
|
|
129
|
+
),
|
|
130
|
+
) -> Iterable[TDataItem]:
|
|
131
|
+
yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
|
|
132
|
+
|
|
133
|
+
return (
|
|
134
|
+
user,
|
|
135
|
+
user_role,
|
|
136
|
+
opportunity,
|
|
137
|
+
opportunity_line_item,
|
|
138
|
+
opportunity_contact_role,
|
|
139
|
+
account,
|
|
140
|
+
contact,
|
|
141
|
+
lead,
|
|
142
|
+
campaign,
|
|
143
|
+
campaign_member,
|
|
144
|
+
product,
|
|
145
|
+
pricebook,
|
|
146
|
+
pricebook_entry,
|
|
147
|
+
task,
|
|
148
|
+
event,
|
|
149
|
+
)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Salesforce source helpers"""
|
|
2
|
+
|
|
3
|
+
from typing import Iterable, Optional
|
|
4
|
+
|
|
5
|
+
import pendulum
|
|
6
|
+
from dlt.common.typing import TDataItem
|
|
7
|
+
from simple_salesforce import Salesforce
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_records(
|
|
11
|
+
sf: Salesforce,
|
|
12
|
+
sobject: str,
|
|
13
|
+
last_state: Optional[str] = None,
|
|
14
|
+
replication_key: Optional[str] = None,
|
|
15
|
+
) -> Iterable[TDataItem]:
|
|
16
|
+
"""
|
|
17
|
+
Retrieves records from Salesforce for a specified sObject.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
sf (Salesforce): An instance of the Salesforce API client.
|
|
21
|
+
sobject (str): The name of the sObject to retrieve records from.
|
|
22
|
+
last_state (str, optional): The last known state for incremental loading. Defaults to None.
|
|
23
|
+
replication_key (str, optional): The replication key for incremental loading. Defaults to None.
|
|
24
|
+
|
|
25
|
+
Yields:
|
|
26
|
+
Dict[TDataItem]: A dictionary representing a record from the Salesforce sObject.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
# Get all fields for the sobject
|
|
30
|
+
desc = getattr(sf, sobject).describe()
|
|
31
|
+
# Salesforce returns compound fields as separate fields, so we need to filter them out
|
|
32
|
+
compound_fields = {
|
|
33
|
+
f["compoundFieldName"]
|
|
34
|
+
for f in desc["fields"]
|
|
35
|
+
if f["compoundFieldName"] is not None
|
|
36
|
+
} - {"Name"}
|
|
37
|
+
# Salesforce returns datetime fields as timestamps, so we need to convert them
|
|
38
|
+
date_fields = {
|
|
39
|
+
f["name"] for f in desc["fields"] if f["type"] in ("datetime",) and f["name"]
|
|
40
|
+
}
|
|
41
|
+
# If no fields are specified, use all fields except compound fields
|
|
42
|
+
fields = [f["name"] for f in desc["fields"] if f["name"] not in compound_fields]
|
|
43
|
+
|
|
44
|
+
# Generate a predicate to filter records by the replication key
|
|
45
|
+
predicate, order_by, n_records = "", "", 0
|
|
46
|
+
if replication_key:
|
|
47
|
+
if last_state:
|
|
48
|
+
predicate = f"WHERE {replication_key} > {last_state}"
|
|
49
|
+
order_by = f"ORDER BY {replication_key} ASC"
|
|
50
|
+
query = f"SELECT {', '.join(fields)} FROM {sobject} {predicate} {order_by}"
|
|
51
|
+
|
|
52
|
+
# Query all records in batches
|
|
53
|
+
for page in getattr(sf.bulk, sobject).query_all(query, lazy_operation=True):
|
|
54
|
+
for record in page:
|
|
55
|
+
# Strip out the attributes field
|
|
56
|
+
record.pop("attributes", None)
|
|
57
|
+
for field in date_fields:
|
|
58
|
+
# Convert Salesforce timestamps to ISO 8601
|
|
59
|
+
if record.get(field):
|
|
60
|
+
record[field] = pendulum.from_timestamp(
|
|
61
|
+
record[field] / 1000,
|
|
62
|
+
).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
63
|
+
yield from page
|
|
64
|
+
n_records += len(page)
|
|
@@ -15,7 +15,7 @@ from typing import (
|
|
|
15
15
|
Optional,
|
|
16
16
|
Union,
|
|
17
17
|
)
|
|
18
|
-
from urllib.parse import ParseResult, parse_qs, quote, urlparse
|
|
18
|
+
from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
|
|
19
19
|
|
|
20
20
|
import dlt
|
|
21
21
|
import gcsfs # type: ignore
|
|
@@ -84,6 +84,7 @@ from ingestr.src.linkedin_ads.dimension_time_enum import (
|
|
|
84
84
|
from ingestr.src.mongodb import mongodb_collection
|
|
85
85
|
from ingestr.src.notion import notion_databases
|
|
86
86
|
from ingestr.src.personio import personio_source
|
|
87
|
+
from ingestr.src.salesforce import salesforce_source
|
|
87
88
|
from ingestr.src.shopify import shopify_source
|
|
88
89
|
from ingestr.src.slack import slack_source
|
|
89
90
|
from ingestr.src.sql_database.callbacks import (
|
|
@@ -135,10 +136,46 @@ class SqlSource:
|
|
|
135
136
|
if uri.startswith("mysql://"):
|
|
136
137
|
uri = uri.replace("mysql://", "mysql+pymysql://")
|
|
137
138
|
|
|
139
|
+
# clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
|
|
138
140
|
if uri.startswith("clickhouse://"):
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
parsed_uri = urlparse(uri)
|
|
142
|
+
|
|
143
|
+
username = parsed_uri.username
|
|
144
|
+
if not username:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
"A username is required to connect to the ClickHouse database."
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
password = parsed_uri.password
|
|
150
|
+
if not password:
|
|
151
|
+
raise ValueError(
|
|
152
|
+
"A password is required to authenticate with the ClickHouse database."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
host = parsed_uri.hostname
|
|
156
|
+
if not host:
|
|
157
|
+
raise ValueError(
|
|
158
|
+
"The hostname or IP address of the ClickHouse server is required to establish a connection."
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
port = parsed_uri.port
|
|
162
|
+
if not port:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
"The TCP port of the ClickHouse server is required to establish a connection."
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
query_params = parse_qs(parsed_uri.query)
|
|
168
|
+
|
|
169
|
+
if "http_port" in query_params:
|
|
170
|
+
del query_params["http_port"]
|
|
171
|
+
|
|
172
|
+
if "secure" not in query_params:
|
|
173
|
+
query_params["secure"] = ["1"]
|
|
174
|
+
|
|
175
|
+
uri = parsed_uri._replace(
|
|
176
|
+
scheme="clickhouse+native",
|
|
177
|
+
query=urlencode(query_params, doseq=True),
|
|
178
|
+
).geturl()
|
|
142
179
|
|
|
143
180
|
query_adapters = []
|
|
144
181
|
if kwargs.get("sql_limit"):
|
|
@@ -1754,7 +1791,7 @@ class AppLovinSource:
|
|
|
1754
1791
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1755
1792
|
if kwargs.get("incremental_key") is not None:
|
|
1756
1793
|
raise ValueError(
|
|
1757
|
-
"
|
|
1794
|
+
"Applovin takes care of incrementality on its own, you should not provide incremental_key"
|
|
1758
1795
|
)
|
|
1759
1796
|
|
|
1760
1797
|
parsed_uri = urlparse(uri)
|
|
@@ -1836,6 +1873,34 @@ class ApplovinMaxSource:
|
|
|
1836
1873
|
).with_resources(table)
|
|
1837
1874
|
|
|
1838
1875
|
|
|
1876
|
+
class SalesforceSource:
|
|
1877
|
+
def handles_incrementality(self) -> bool:
|
|
1878
|
+
return True
|
|
1879
|
+
|
|
1880
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1881
|
+
if kwargs.get("incremental_key"):
|
|
1882
|
+
raise ValueError(
|
|
1883
|
+
"Salesforce takes care of incrementality on its own, you should not provide incremental_key"
|
|
1884
|
+
)
|
|
1885
|
+
|
|
1886
|
+
params = parse_qs(urlparse(uri).query)
|
|
1887
|
+
creds = {
|
|
1888
|
+
"username": params.get("username", [None])[0],
|
|
1889
|
+
"password": params.get("password", [None])[0],
|
|
1890
|
+
"token": params.get("token", [None])[0],
|
|
1891
|
+
}
|
|
1892
|
+
for k, v in creds.items():
|
|
1893
|
+
if v is None:
|
|
1894
|
+
raise MissingValueError(k, "Salesforce")
|
|
1895
|
+
|
|
1896
|
+
src = salesforce_source(**creds) # type: ignore
|
|
1897
|
+
|
|
1898
|
+
if table not in src.resources:
|
|
1899
|
+
raise UnsupportedResourceError(table, "Salesforce")
|
|
1900
|
+
|
|
1901
|
+
return src.with_resources(table)
|
|
1902
|
+
|
|
1903
|
+
|
|
1839
1904
|
class PersonioSource:
|
|
1840
1905
|
def handles_incrementality(self) -> bool:
|
|
1841
1906
|
return True
|
|
@@ -1874,7 +1939,7 @@ class PersonioSource:
|
|
|
1874
1939
|
"custom_reports_list",
|
|
1875
1940
|
]:
|
|
1876
1941
|
raise UnsupportedResourceError(table, "Personio")
|
|
1877
|
-
|
|
1942
|
+
|
|
1878
1943
|
return personio_source(
|
|
1879
1944
|
client_id=client_id[0],
|
|
1880
1945
|
client_secret=client_secret[0],
|
|
@@ -1,24 +1,23 @@
|
|
|
1
1
|
asana==3.2.3
|
|
2
|
-
confluent-kafka>=2.
|
|
2
|
+
confluent-kafka>=2.8.0
|
|
3
3
|
databricks-sql-connector==2.9.3
|
|
4
4
|
dataclasses-json==0.6.7
|
|
5
|
-
dlt==1.
|
|
6
|
-
duckdb_engine==0.
|
|
7
|
-
duckdb==1.
|
|
5
|
+
dlt==1.6.1
|
|
6
|
+
duckdb_engine==0.15.0
|
|
7
|
+
duckdb==1.2.0
|
|
8
8
|
google-ads==25.1.0
|
|
9
9
|
facebook-business==20.0.0
|
|
10
|
-
google-analytics-data==0.18.16
|
|
11
10
|
google-api-python-client==2.130.0
|
|
12
11
|
google-cloud-bigquery-storage==2.24.0
|
|
13
|
-
mysql-connector-python==9.
|
|
12
|
+
mysql-connector-python==9.2.0
|
|
14
13
|
pendulum==3.0.0
|
|
15
14
|
psutil==6.1.1
|
|
16
15
|
psycopg2-binary==2.9.10
|
|
17
16
|
py-machineid==0.6.0
|
|
18
17
|
pyairtable==2.3.3
|
|
19
18
|
pyarrow==18.1.0
|
|
20
|
-
pyathena==3.
|
|
21
|
-
pymongo==4.
|
|
19
|
+
pyathena==3.12.2
|
|
20
|
+
pymongo==4.11.1
|
|
22
21
|
pymysql==1.1.1
|
|
23
22
|
pyrate-limiter==3.7.0
|
|
24
23
|
redshift-connector==2.1.5
|
|
@@ -26,19 +25,18 @@ rich==13.9.4
|
|
|
26
25
|
rudder-sdk-python==2.1.4
|
|
27
26
|
s3fs==2024.10.0
|
|
28
27
|
snowflake-sqlalchemy==1.6.1
|
|
29
|
-
sqlalchemy-bigquery==1.12.
|
|
28
|
+
sqlalchemy-bigquery==1.12.1
|
|
30
29
|
sqlalchemy-hana==2.0.0
|
|
31
30
|
sqlalchemy-redshift==0.8.14
|
|
32
31
|
SQLAlchemy==1.4.52
|
|
33
32
|
sqlalchemy2-stubs==0.0.2a38
|
|
34
33
|
stripe==10.7.0
|
|
35
|
-
tqdm==4.67.
|
|
34
|
+
tqdm==4.67.1
|
|
36
35
|
typer==0.13.1
|
|
37
36
|
types-requests==2.32.0.20240907
|
|
38
37
|
s3fs==2024.10.0
|
|
39
38
|
pyarrow==18.1.0
|
|
40
|
-
|
|
41
|
-
google-analytics-data==0.18.16
|
|
39
|
+
google-analytics-data==0.18.17
|
|
42
40
|
asana==3.2.3
|
|
43
41
|
flatten_json==0.1.14
|
|
44
42
|
dataclasses-json==0.6.7
|
|
@@ -46,3 +44,4 @@ gcsfs==2024.10.0
|
|
|
46
44
|
clickhouse-connect==0.8.14
|
|
47
45
|
clickhouse-driver==0.2.9
|
|
48
46
|
clickhouse-sqlalchemy==0.2.7
|
|
47
|
+
simple-salesforce==1.12.6
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
version = "v0.13.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|