ingestr 0.14.1__tar.gz → 0.14.3__tar.gz

ingestr-0.14.3/.github/workflows/release.yml

@@ -0,0 +1,151 @@
+# WARN: this is used as a notification filter in our slack.
+# When updating this, make sure to update the workflow subscription as well.
+name: "Release ingestr on pip and ghcr.io"
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/${{ github.repository }}
+
+jobs:
+  tests:
+    uses: ./.github/workflows/tests.yml
+  pip-release:
+    runs-on: ubuntu-latest
+    needs: tests
+    env:
+      TWINE_PASSWORD: ${{ secrets.PYPI_INGESTR_TOKEN }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+        
+        # github actions invoke a fresh shell
+        # for each step. This causes us to lose our
+        # venv context. So we run all our build commands
+        # in a single step to simplify our pipeline.
+      - name: build and release
+        run: |
+          pip install uv
+          uv venv
+          source .venv/bin/activate
+          uv pip install -r requirements-dev.txt
+          make build
+          make upload-release
+
+  docker-build:
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-24.04-arm
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        id: build
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          tags: ${{ env.IMAGE }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
+      
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          # The following strips the 'sha256:' prefix from the digest using bash parameter expansion.
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+    
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  docker-publish:
+    permissions:
+      contents: read
+      packages: write
+
+    runs-on: ubuntu-latest
+    needs:
+      - docker-build
+      - tests
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern={{raw}}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
+
+

{ingestr-0.14.1 → ingestr-0.14.3}/Dockerfile

@@ -26,7 +26,7 @@ RUN \
   export ACCEPT_EULA='Y' && \
   # Install build dependencies
   apt-get update && \
-  apt-get install -y curl gcc gpg libpq-dev build-essential unixodbc-dev g++ apt-transport-https
+  apt-get install -y curl gcc gpg libpq-dev build-essential unixodbc-dev g++ apt-transport-https git
 
 RUN \ 
   # Install pyodbc db drivers for MSSQL and PostgreSQL
@@ -50,6 +50,9 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then \
     cp /app/requirements_arm64.txt /app/requirements.txt; \
 fi
 
+# Generate version file
+RUN make write-build-info
+
 # Install all required packages and the application.
 RUN uv pip install --requirement requirements.txt pyodbc .
 

{ingestr-0.14.1 → ingestr-0.14.3}/Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 .ONESHELL:
-.PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell
+.PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell write-build-info remove-build-info
 
 BUILDINFO=ingestr/src/buildinfo.py
 
@@ -46,8 +46,14 @@ lint-docs:
 tl: test lint
 
 build: lock-deps
-	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
+	$(MAKE) write-build-info
 	rm -rf dist && python3 -m build
+	$(MAKE) remove-build-info
+
+write-build-info:
+	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
+
+remove-build-info:
 	rm -f ${BUILDINFO}
 
 upload-release:

{ingestr-0.14.1 → ingestr-0.14.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.14.1
+Version: 0.14.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.14.1 → ingestr-0.14.3}/docs/.vitepress/config.mjs

@@ -5,6 +5,17 @@ export default defineConfig({
   title: "ingestr",
   description: "Ingest & copy data between any source and any destination",
   base: "/ingestr/",
+  transformPageData(pageData) {
+    const canonicalUrl = `https://getbruin.com/docs/ingestr/${pageData.relativePath}`
+      .replace(/index\.md$/, '')
+      .replace(/\.md$/, '.html')
+
+    pageData.frontmatter.head ??= []
+    pageData.frontmatter.head.push([
+      'link',
+      { rel: 'canonical', href: canonicalUrl }
+    ])
+  },
   head: [
     [
       "script",
@@ -141,6 +152,7 @@ export default defineConfig({
               { text: "Gorgias", link: "/supported-sources/gorgias.md" },
               { text: "HubSpot", link: "/supported-sources/hubspot.md" },
               { text: "Internet Society Pulse", link: "/supported-sources/isoc-pulse.md" },
+              { text: "Jira", link: "/supported-sources/jira.md" },
               { text: "Klaviyo", link: "/supported-sources/klaviyo.md" },
               { text: "Linear", link: "/supported-sources/linear.md" },
               { text: "LinkedIn Ads", link: "/supported-sources/linkedin_ads.md" },

{ingestr-0.14.1 → ingestr-0.14.3}/docs/index.md

@@ -7,7 +7,7 @@ hero:
   text: Copy data between any source and any destination
   tagline: "ingestr is a command-line application that allows copying data from any source into any destination database."
   image:
-    src: https://github.com/bruin-data/ingestr/blob/main/resources/demo.gif?raw=true
+    src: /demo.gif
     alt: ingestr logo
   actions:
     - theme: brand

ingestr-0.14.3/docs/supported-sources/elasticsearch.md

@@ -0,0 +1,80 @@
+# Elasticsearch
+[Elasticsearch](https://www.elastic.co/elasticsearch) is a distributed, RESTful search and analytics engine designed for fast and scalable data retrieval.
+
+ingestr supports Elasticsearch as both a source and destination.
+
+## URI format
+The URI format for Elasticsearch is as follows:
+
+```plaintext
+elasticsearch://username:password@host:port?secure=<secure>&verify_certs=<verify_certs>
+```
+
+URI parameters:
+- `username`: The username used to authenticate with Elasticsearch (optional).
+- `password`: The password associated with the specified username (optional).
+- `host`: The host address of the Elasticsearch server.
+- `port`: The port number used by the Elasticsearch server (default: 9200).
+- `secure`: Enables HTTPS when set to true. By default, it is true. Set to false for HTTP connections.
+- `verify_certs`: Verifies TLS certificates when set to true. By default, it is true. Only used for sources.
+
+The same URI structure can be used both for sources and destinations.
+
+## Using Elasticsearch as a source
+
+### Source Table
+
+`<index-name>`: Fetches all available documents from the specified index.
+
+```bash
+ingestr ingest \
+    --source-uri "elasticsearch://elastic:changeme@localhost:9200?secure=false&verify_certs=false" \
+    --source-table 'test-index' \
+    --dest-uri "duckdb:///users.duckdb" \
+    --dest-table 'dest.users_detail'
+```
+
+This command retrieves all documents from the test-index in Elasticsearch and copy them to the dest.users_detail table in DuckDB.
+
+<img alt="ingestion_elasticsearch_img" src="../media/ingestion_elasticsearch_img.png" />
+
+## Using Elasticsearch as a destination
+
+Elasticsearch can be used as a destination to load data from various sources. The `--dest-table` option specifies the index name where data will be loaded.
+
+### Elasticsearch Cloud (with authentication)
+
+```bash
+ingestr ingest \
+  --source-uri "postgres://user:pass@localhost:5432/mydb" \
+  --source-table "public.users" \
+  --dest-uri "elasticsearch://username:password@cluster.cloud.es.io:443" \
+  --dest-table "users_index"
+```
+
+> [!NOTE]
+> Cloud Elasticsearch instances typically use HTTPS (port 443). The `secure` parameter defaults to `true`, so HTTPS is used automatically.
+
+### Local Elasticsearch with authentication
+
+```bash
+ingestr ingest \
+  --source-uri "csv:///path/to/data.csv" \
+  --source-table "data" \
+  --dest-uri "elasticsearch://elastic:changeme@localhost:9200?secure=false" \
+  --dest-table "myindex"
+```
+
+### Local Elasticsearch without authentication
+
+```bash
+ingestr ingest \
+  --source-uri "csv:///path/to/data.csv" \
+  --source-table "data" \
+  --dest-uri "elasticsearch://localhost:9200?secure=false" \
+  --dest-table "myindex"
+```
+
+> [!TIP]
+> By default, ingestr uses a "replace" strategy which deletes the existing index before loading new data. The target index will be created automatically if it doesn't exist.
+

{ingestr-0.14.1 → ingestr-0.14.3}/docs/supported-sources/fluxx.md

@@ -104,6 +104,7 @@ Fluxx source allows ingesting the following sources into separate tables:
 | `affiliate_type` | id | - | replace | Affiliate Type management and tracking |
 | `aha_requirements_tickets` | id | - | replace | Aha Requirements Tickets management and tracking |
 | `alert` | id | - | replace | Alert management and tracking |
+| `alert_email` | id | - | replace | Alert email configurations and templates for notification delivery |
 | `alert_email_user` | id | - | replace | Alert Email User management and tracking |
 | `alert_model_log` | id | - | replace | Alert Model Log management and tracking |
 | `alert_recipient` | id | - | replace | Alert Recipient management and tracking |

ingestr-0.14.3/docs/supported-sources/intercom.md

@@ -0,0 +1,57 @@
+# Intercom
+
+[Intercom](https://www.intercom.com/) is a customer messaging platform that helps businesses connect with customers through targeted, behavior-driven messages.
+
+ingestr supports Intercom as a source.
+
+## URI format
+
+The URI format for Intercom is as follows:
+
+```plaintext
+intercom://?access_token=<access_token>&region=<region>
+```
+
+URI parameters:
+
+- `access_token`: The access token is used for authentication with the Intercom API.
+- `region`: The data region where your Intercom workspace is hosted. Valid values are `us`, `eu`, or `au`. Defaults to `us`.
+
+The URI is used to connect to the Intercom API for extracting data.
+
+## Setting up an Intercom Integration
+
+Intercom requires a few steps to set up an integration, please follow the guide dltHub [has built here](https://dlthub.com/docs/dlt-ecosystem/verified-sources/intercom).
+
+Once you complete the guide, you should have an access token. Let's say your access token is `dG9rOjE...` and your workspace is in the US region, here's a sample command that will copy the data from Intercom into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'intercom://?access_token=dG9rOjE...&region=us' \
+    --source-table 'contacts' \
+    --dest-uri duckdb:///intercom.duckdb \
+    --dest-table 'intercom.contacts'
+```
+
+The result of this command will be a table in the `intercom.duckdb` database.
+
+## Tables
+
+Intercom source allows ingesting the following sources into separate tables:
+
+| Table | PK | Inc Key | Inc Strategy | Details |
+|-------|----|---------|--------------|---------|
+| [contacts](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Contacts/contact/) | id | updated_at | merge | Retrieves information about contacts (visitors, users, and leads) |
+| [companies](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Companies/company/) | id | updated_at | merge | Retrieves information about companies |
+| [conversations](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Conversations/conversation/) | id | updated_at | merge | Retrieves conversation data |
+| [articles](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Articles/article/) | id | updated_at | merge | Retrieves help center articles |
+| [tags](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Tags/tag/) | id | - | replace | Retrieves tags used to organize contacts and companies |
+| [segments](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Segments/segment/) | id | - | replace | Retrieves segments for filtering contacts and companies |
+| [admins](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Admins/admin/) | id | - | replace | Retrieves admin user information |
+| [teams](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Teams/team/) | id | - | replace | Retrieves team information |
+| [data_attributes](https://developers.intercom.com/docs/references/rest-api/api.intercom.io/Data%20Attributes/dataattribute/) | name | - | replace | Retrieves data attributes (both built-in and custom attributes) |
+
+Use these as `--source-table` parameter in the `ingestr ingest` command.
+
+> [!TIP]
+> Resources marked with "merge" Inc Strategy support incremental loading based on the `updated_at` timestamp, which means subsequent runs will only fetch new or updated records.

ingestr-0.14.3/docs/supported-sources/jira.md

@@ -0,0 +1,61 @@
+# Jira
+
+[Jira](https://www.atlassian.com/software/jira) is a proprietary issue tracking product developed by Atlassian that allows bug tracking and agile project management.
+
+ingestr supports Jira as a source through [Jira's REST API v3](https://developer.atlassian.com/cloud/jira/platform/rest/v3/).
+
+## URI format
+
+The URI format for Jira is:
+
+```plaintext
+jira://your-domain.atlassian.net?email=<email>&api_token=<api_token>
+```
+
+URI parameters:
+- `email`: the email address used for authentication with the Jira API
+- `api_token`: the API token for authentication (required for Jira Cloud)
+
+## Example usage
+
+Assuming your Jira domain is `company.atlassian.net`, email is `user@company.com`, and API token is `ATATT3xFfGF0...`, you can ingest issues into DuckDB using:
+
+```bash
+ingestr ingest \
+  --source-uri 'jira://company.atlassian.net?email=user@company.com&api_token=ATATT3xFfGF0...' \
+  --source-table 'issues' \
+  --dest-uri duckdb:///jira.duckdb \
+  --dest-table 'dest.issues'
+```
+
+## Authentication
+
+To connect to Jira, you need:
+
+1. **Email**: Your Atlassian account email
+2. **API Token**: Create one from your [Atlassian Account Settings](https://id.atlassian.com/manage-profile/security/api-tokens)
+
+## Tables
+
+Jira source allows ingesting the following tables:
+
+| Table | PK | Inc Key | Inc Strategy | Details |
+| ----- | -- | ------- | ------------ | ------- |
+| `projects` | id | - | replace | Fetches all projects from your Jira instance. |
+| `issues` | id | fields.updated | merge | Fetches all issues with support for incremental loading based on updated timestamp. |
+| `users` | accountId | - | replace | Fetches users from your Jira instance. |
+| `issue_types` | id | - | replace | Fetches all issue types configured in your Jira instance. |
+| `statuses` | id | - | replace | Fetches all workflow statuses from your Jira instance. |
+| `priorities` | id | - | replace | Fetches all issue priorities from your Jira instance. |
+| `resolutions` | id | - | replace | Fetches all issue resolutions from your Jira instance. |
+| `project_versions` | id | - | replace | Fetches versions for each project. |
+| `project_components` | id | - | replace | Fetches components for each project. |
+
+Use these as the `--source-table` parameter in the `ingestr ingest` command.
+
+## Incremental Loading
+
+The `issues` table supports incremental loading based on the `updated` field. This means subsequent runs will only fetch issues that have been modified since the last run, making the data ingestion more efficient for large Jira instances.
+
+> [!NOTE]
+> Most tables use "replace" write disposition, meaning they will overwrite existing data on each run. Only the `issues` table supports incremental loading with "merge" disposition.

{ingestr-0.14.1 → ingestr-0.14.3}/docs/supported-sources/mongodb.md

@@ -1,11 +1,13 @@
 # MongoDB
 MongoDB is a popular, open source NoSQL database known for its flexibility, scalability, and wide adoption in a variety of applications.
 
-ingestr supports MongoDB as a source.
+ingestr supports MongoDB as both a source and destination.
 
 ## URI format
-The URI format for MongoDB is as follows:
 
+MongoDB supports two connection string formats:
+
+### Standard format (local/self-hosted)
 ```plaintext
 mongodb://user:password@host:port
 ```
@@ -16,12 +18,21 @@ URI parameters:
 - `host`: the host address of the database server
 - `port`: the port number the database server is listening on, default is 27017 for MongoDB
 
+### SRV format (MongoDB Atlas)
+```plaintext
+mongodb+srv://user:password@cluster.xxxxx.mongodb.net/?retryWrites=true&w=majority
+```
 
-> [!CAUTION]
-> Do not put the database name at the end of the URI for MongoDB, instead make it a part of `--source-table` option as `database.collection` format.
+URI parameters:
+- `user`: the user name to connect to the database
+- `password`: the password for the user
+- `cluster.xxxxx.mongodb.net`: the cluster hostname provided by MongoDB Atlas
+- Query parameters like `retryWrites` and `w` are optional but recommended for Atlas connections
 
+> [!CAUTION]
+> Do not put the database name at the end of the URI for MongoDB, instead make it a part of `--source-table` or `--dest-table` option as `database.collection` format.
 
-You can read more about MongoDB's connection string format [here](https://docs.mongodb.com/manual/reference/connection-string/).
+The same URI structure can be used both for sources and destinations. You can read more about MongoDB's connection string format [here](https://docs.mongodb.com/manual/reference/connection-string/).
 
 ## Source table format
 
@@ -148,3 +159,43 @@ ingestr performs several validations on custom aggregation pipelines:
 - Add appropriate indexes to support your aggregation pipeline
 - Consider using `$limit` to restrict the number of documents processed
 - For large datasets, MongoDB's `allowDiskUse: true` option is automatically enabled for aggregation pipelines
+
+## Using MongoDB as a destination
+
+MongoDB can be used as a destination to load data from various sources. The `--dest-table` option follows the same format: `database.collection`.
+
+### MongoDB Atlas
+
+```bash
+ingestr ingest \
+  --source-uri "postgres://user:pass@localhost:5432/mydb" \
+  --source-table "public.users" \
+  --dest-uri "mongodb+srv://username:password@cluster0.xxxxx.mongodb.net/?retryWrites=true&w=majority" \
+  --dest-table "mydb.users"
+```
+
+> [!NOTE]
+> When using MongoDB Atlas as a destination, ensure your IP address is whitelisted in Network Access settings.
+
+### Local MongoDB with authentication
+
+```bash
+ingestr ingest \
+  --source-uri "csv:///path/to/data.csv" \
+  --source-table "data" \
+  --dest-uri "mongodb://username:password@localhost:27017/?authSource=admin" \
+  --dest-table "mydb.mycollection"
+```
+
+### Local MongoDB without authentication
+
+```bash
+ingestr ingest \
+  --source-uri "csv:///path/to/data.csv" \
+  --source-table "data" \
+  --dest-uri "mongodb://localhost:27017" \
+  --dest-table "mydb.mycollection"
+```
+
+> [!TIP]
+> By default, ingestr uses a "replace" strategy which deletes existing data in the collection before loading new data. The target database and collection will be created automatically if they don't exist.

ingestr-0.14.3/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.14.3"

{ingestr-0.14.1 → ingestr-0.14.3}/ingestr/src/destinations.py

@@ -813,30 +813,7 @@ class ElasticsearchDestination:
 
 class MongoDBDestination:
     def dlt_dest(self, uri: str, **kwargs):
-        from urllib.parse import urlparse
-
-        parsed_uri = urlparse(uri)
-
-        # Extract connection details from URI
-        host = parsed_uri.hostname or "localhost"
-        port = parsed_uri.port or 27017
-        username = parsed_uri.username
-        password = parsed_uri.password
-        database = (
-            parsed_uri.path.lstrip("/") if parsed_uri.path.lstrip("/") else "ingestr_db"
-        )
-
-        # Build connection string
-        if username and password:
-            connection_string = f"mongodb://{username}:{password}@{host}:{port}"
-        else:
-            connection_string = f"mongodb://{host}:{port}"
-
-        # Add query parameters if any
-        if parsed_uri.query:
-            connection_string += f"?{parsed_uri.query}"
-
-        return mongodb_insert(connection_string, database)
+        return mongodb_insert(uri)
 
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
         return {

{ingestr-0.14.1 → ingestr-0.14.3}/ingestr/src/elasticsearch/helpers.py

@@ -1,6 +1,7 @@
 """Elasticsearch destination helpers"""
 
 import json
+import logging
 from typing import Any, Dict, Iterator
 from urllib.parse import urlparse
 
@@ -9,6 +10,10 @@ import dlt
 from elasticsearch import Elasticsearch
 from elasticsearch.helpers import bulk
 
+# Suppress Elasticsearch transport logging
+logging.getLogger("elasticsearch.transport").setLevel(logging.WARNING)
+logging.getLogger("elastic_transport.transport").setLevel(logging.WARNING)
+
 
 def process_file_items(file_path: str) -> Iterator[Dict[str, Any]]:
     """Process items from a file path (JSONL format)."""
@@ -52,15 +57,36 @@ def elasticsearch_insert(
     parsed = urlparse(connection_string)
 
     # Build Elasticsearch client configuration
-    hosts = [
-        {
-            "host": parsed.hostname or "localhost",
-            "port": parsed.port or 9200,
-            "scheme": parsed.scheme or "http",
-        }
-    ]
-
-    es_config: Dict[str, Any] = {"hosts": hosts}
+    actual_url = connection_string
+    secure = True  # Default to HTTPS (secure by default)
+
+    if connection_string.startswith("elasticsearch://"):
+        actual_url = connection_string.replace("elasticsearch://", "")
+
+        # Parse to check for query parameters
+        temp_parsed = urlparse("http://" + actual_url)
+        from urllib.parse import parse_qs
+
+        query_params = parse_qs(temp_parsed.query)
+
+        # Check ?secure parameter (defaults to true)
+        if "secure" in query_params:
+            secure = query_params["secure"][0].lower() in ["true", "1", "yes"]
+
+        # Remove query params from URL for ES client
+        actual_url = actual_url.split("?")[0]
+
+        # Add scheme
+        scheme = "https" if secure else "http"
+        actual_url = f"{scheme}://{actual_url}"
+
+        parsed = urlparse(actual_url)
+
+    es_config: Dict[str, Any] = {
+        "hosts": [actual_url],
+        "verify_certs": secure,
+        "ssl_show_warn": False,
+    }
 
     # Add authentication if present
     if parsed.username and parsed.password:

{ingestr-0.14.1 → ingestr-0.14.3}/ingestr/src/factory.py

@@ -54,7 +54,9 @@ from ingestr.src.sources import (
     GorgiasSource,
     HubspotSource,
     InfluxDBSource,
+    IntercomSource,
     IsocPulseSource,
+    JiraSource,
     KafkaSource,
     KinesisSource,
     KlaviyoSource,
@@ -166,6 +168,8 @@ class SourceDestinationFactory:
         "fluxx": FluxxSource,
         "slack": SlackSource,
         "hubspot": HubspotSource,
+        "intercom": IntercomSource,
+        "jira": JiraSource,
         "airtable": AirtableSource,
         "klaviyo": KlaviyoSource,
         "mixpanel": MixpanelSource,

{ingestr-0.14.1 → ingestr-0.14.3}/ingestr/src/fluxx/__init__.py

@@ -96,6 +96,15 @@ FLUXX_RESOURCES = {
             "workflow_events": {"data_type": "json", "field_type": "relation"},
         },
     },
+    "alert_email": {
+        "endpoint": "alert_email",
+        "fields": {
+            "alert_id": {"data_type": "bigint", "field_type": "column"},
+            "created_at": {"data_type": "timestamp", "field_type": "column"},
+            "id": {"data_type": "bigint", "field_type": "column"},
+            "updated_at": {"data_type": "timestamp", "field_type": "column"},
+        },
+    },
     "affiliate": {
         "endpoint": "affiliate",
         "fields": {