ingestr 0.13.3__tar.gz → 0.13.5__tar.gz

ingestr-0.13.5/.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: "Release ingestr on pip and ghcr.io"
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  tests:
+    uses: ./.github/workflows/tests.yml
+  pip-release:
+    runs-on: ubuntu-latest
+    needs: tests
+    env:
+      TWINE_PASSWORD: ${{ secrets.PYPI_INGESTR_TOKEN }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+        
+        # github actions invoke a fresh shell
+        # for each step. This causes us to lose our
+        # venv context. So we run all our build commands
+        # in a single step to simplify our pipeline.
+      - name: build and release
+        run: |
+          pip install uv
+          uv venv
+          source .venv/bin/activate
+          uv pip install -r requirements-dev.txt
+          make build
+          make upload-release
+
+  docker-release:
+    needs: tests
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm64/v8
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+    
+

{ingestr-0.13.3 → ingestr-0.13.5}/.github/workflows/tests.yml

@@ -2,9 +2,8 @@ on:
   push:
     branches:
       - main
-    tags:
-      - "*"
   workflow_dispatch: {}
+  workflow_call: {}
   pull_request:
     branches:
       - main
@@ -13,11 +12,6 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-
-
 jobs:
   tests:
     strategy:
@@ -70,39 +64,3 @@ jobs:
         run: make test-ci
       - name: check the formatting
         run: make lint-ci
-
-  build-and-push-image:
-    needs: tests
-    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push Docker image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64,linux/arm64/v8
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max

{ingestr-0.13.3 → ingestr-0.13.5}/.gitignore

@@ -20,3 +20,4 @@ node_modules
 *.db
 *.db.wal
 *.sql
+ingestr/src/buildinfo.py

{ingestr-0.13.3 → ingestr-0.13.5}/Makefile

@@ -1,6 +1,9 @@
+SHELL := /bin/bash
 .ONESHELL:
 .PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell
 
+BUILDINFO=ingestr/src/buildinfo.py
+
 venv: venv/touchfile
 
 venv/touchfile: requirements-dev.txt requirements.txt
@@ -36,7 +39,9 @@ lint-docs:
 tl: test lint
 
 build:
+	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
 	rm -rf dist && python3 -m build
+	rm -f ${BUILDINFO}
 
 upload-release:
 	twine upload --verbose dist/*

{ingestr-0.13.3 → ingestr-0.13.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.3
+Version: 0.13.5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -132,7 +132,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
 > After cloning `ingestr` make sure to run `make setup` to install githooks.
 
 ## Supported sources & destinations
-
 <table>
     <tr>
         <th></th>
@@ -142,33 +141,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
     </tr>
-    <tr>
-        <td>Postgres</td>
-        <td>✅</td>
-        <td>✅</td>
-    </tr>
     <tr>
         <td>BigQuery</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Snowflake</td>
-        <td>✅</td>
+        <td>ClickHouse</td>
+        <td>❌</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Redshift</td>
+        <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Databricks</td>
+        <td>DuckDB</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>DuckDB</td>
+        <td>Local CSV file</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
@@ -178,12 +172,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
     </tr>
     <tr>
-        <td>Local CSV file</td>
-        <td>✅</td>
+        <td>MongoDB</td>
         <td>✅</td>
+        <td>❌</td>
     </tr>
     <tr>
-        <td>MongoDB</td>
+        <td>MySQL</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -192,18 +186,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>SAP Hana</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
     <tr>
-        <td>SQLite</td>
+        <td>Snowflake</td>
+        <td>✅</td>
         <td>✅</td>
-        <td>❌</td>
     </tr>
     <tr>
-        <td>MySQL</td>
+        <td>SQLite</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -218,11 +222,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Apache Kafka</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>App Store</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Asana</td>
         <td>✅</td>
@@ -243,6 +257,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Github</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -254,7 +273,17 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>HubSpot</td>
+        <td>Google Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Google Analytics</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Intercom</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -263,6 +292,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>LinkedIn Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
@@ -288,6 +322,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>TikTok Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Zendesk</td>
         <td>✅</td>

{ingestr-0.13.3 → ingestr-0.13.5}/README.md

@@ -68,7 +68,6 @@ Pull requests are welcome. However, please open an issue first to discuss what y
 > After cloning `ingestr` make sure to run `make setup` to install githooks.
 
 ## Supported sources & destinations
-
 <table>
     <tr>
         <th></th>
@@ -78,33 +77,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Databases</strong></td>
     </tr>
-    <tr>
-        <td>Postgres</td>
-        <td>✅</td>
-        <td>✅</td>
-    </tr>
     <tr>
         <td>BigQuery</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Snowflake</td>
-        <td>✅</td>
+        <td>ClickHouse</td>
+        <td>❌</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Redshift</td>
+        <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>Databricks</td>
+        <td>DuckDB</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
     <tr>
-        <td>DuckDB</td>
+        <td>Local CSV file</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
@@ -114,12 +108,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
     </tr>
     <tr>
-        <td>Local CSV file</td>
-        <td>✅</td>
+        <td>MongoDB</td>
         <td>✅</td>
+        <td>❌</td>
     </tr>
     <tr>
-        <td>MongoDB</td>
+        <td>MySQL</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -128,18 +122,28 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Postgres</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>Redshift</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>SAP Hana</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
     <tr>
-        <td>SQLite</td>
+        <td>Snowflake</td>
+        <td>✅</td>
         <td>✅</td>
-        <td>❌</td>
     </tr>
     <tr>
-        <td>MySQL</td>
+        <td>SQLite</td>
         <td>✅</td>
         <td>❌</td>
     </tr>
@@ -154,11 +158,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Apache Kafka</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>AppsFlyer</td>
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>App Store</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Asana</td>
         <td>✅</td>
@@ -179,6 +193,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Github</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Gorgias</td>
         <td>✅</td>
@@ -190,7 +209,17 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>HubSpot</td>
+        <td>Google Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Google Analytics</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Intercom</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -199,6 +228,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>LinkedIn Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Notion</td>
         <td>✅</td>
@@ -224,6 +258,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>TikTok Ads</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Zendesk</td>
         <td>✅</td>

{ingestr-0.13.3 → ingestr-0.13.5}/docs/supported-sources/clickhouse.md

@@ -1,20 +1,27 @@
 # ClickHouse
 ClickHouse is a fast, open-source, column-oriented database management system that allows for high performance data ingestion and querying.
 
-Ingestr supports ClickHouse as a destination.
+ingestr supports ClickHouse as a source and destination.
 
 ## URI format
+The URI format for ClickHouse as a source is as follows:
+
+```plaintext
+clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
+```
+
 The URI format for ClickHouse as a destination is as follows:
 
 ```plaintext
 clickhouse://<username>:<password>@<host>:<port>?http_port=<http_port>&secure=<secure>
 ```
+
 ## URI parameters:
 - `username` (required): The username is required to authenticate with the ClickHouse server.
 - `password` (required): The password is required to authenticate the provided username.
 - `host` (required): The hostname or IP address of the ClickHouse server where the database is hosted.
 - `port` (required): The TCP port number used by the ClickHouse server.
-- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface.By default, it is set to port `8443`.
+- `http_port` (optional): The port number to use when connecting to the ClickHouse server's HTTP interface. By default, it is set to port `8443`. It should only be used when ClickHouse is the destination. 
 - `secure` (optional): Set to `1` for a secure HTTPS connection or `0` for a non-secure HTTP connection. By default, it is set to `1`.
 
 ClickHouse requires a `username`, `password`, `host` and `port` to connect to the ClickHouse server. For more information, read [here](https://dlthub.com/docs/dlt-ecosystem/destinations/clickhouse#2-setup-clickhouse-database). Once you've completed the guide, you should have all the above-mentioned credentials.

{ingestr-0.13.3 → ingestr-0.13.5}/ingestr/src/destinations.py

@@ -7,12 +7,13 @@ import tempfile
 from urllib.parse import parse_qs, quote, urlparse
 
 import dlt
-import pyarrow.parquet  # type: ignore
 from dlt.common.configuration.specs import AwsCredentials
 from dlt.destinations.impl.clickhouse.configuration import (
     ClickHouseCredentials,
 )
 
+from ingestr.src.loader import load_dlt_file
+
 
 class GenericSqlDestination:
     def dlt_run_params(self, uri: str, table: str, **kwargs) -> dict:
@@ -59,9 +60,14 @@ class BigQueryDestination:
                 base64.b64decode(credentials_base64[0]).decode("utf-8")
             )
 
+        project_id = None
+        if source_fields.hostname:
+            project_id = source_fields.hostname
+
         return dlt.destinations.bigquery(
             credentials=credentials,  # type: ignore
             location=location,
+            project_id=project_id,
             **kwargs,
         )
 
@@ -184,11 +190,9 @@ class CsvDestination(GenericSqlDestination):
         if output_path.count("/") > 1:
             os.makedirs(os.path.dirname(output_path), exist_ok=True)
 
-        table = pyarrow.parquet.read_table(first_file_path)
-        rows = table.to_pylist()
         with open(output_path, "w", newline="") as csv_file:
             csv_writer = None
-            for row in rows:
+            for row in load_dlt_file(first_file_path):
                 row = filter_keys(row)
                 if csv_writer is None:
                     csv_writer = csv.DictWriter(csv_file, fieldnames=row.keys())

{ingestr-0.13.3 → ingestr-0.13.5}/ingestr/src/factory.py

@@ -66,6 +66,7 @@ SQL_SOURCE_SCHEMES = [
     "oracle",
     "oracle+cx_oracle",
     "hana",
+    "clickhouse",
 ]
 
 

ingestr-0.13.5/ingestr/src/loader.py

@@ -0,0 +1,69 @@
+import csv
+import gzip
+import json
+import subprocess
+from contextlib import contextmanager
+from typing import Generator
+
+from pyarrow.parquet import ParquetFile  # type: ignore
+
+PARQUET_BATCH_SIZE = 64
+
+
+class UnsupportedLoaderFileFormat(Exception):
+    pass
+
+
+def load_dlt_file(filepath: str) -> Generator:
+    """
+    load_dlt_file reads dlt loader files. It handles different loader file formats
+    automatically. It returns a generator that yield data items as a python dict
+    """
+    result = subprocess.run(
+        ["file", "-b", filepath],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+
+    filetype = result.stdout.strip()
+    with factory(filetype, filepath) as reader:
+        yield from reader
+
+
+def factory(filetype: str, filepath: str):
+    # ???(turtledev): can dlt produce non-gizpped jsonl files?
+    if filetype.startswith("gzip"):
+        return jsonlfile(filepath)
+    elif filetype.startswith("CSV"):
+        return csvfile(filepath)
+    elif filetype.startswith("Apache Parquet"):
+        return parquetfile(filepath)
+    else:
+        raise UnsupportedLoaderFileFormat(filetype)
+
+
+@contextmanager
+def jsonlfile(filepath: str):
+    def reader(fd):
+        for line in fd:
+            yield json.loads(line.decode().strip())
+
+    with gzip.open(filepath) as fd:
+        yield reader(fd)
+
+
+@contextmanager
+def csvfile(filepath: str):
+    with open(filepath, "r") as fd:
+        yield csv.DictReader(fd)
+
+
+@contextmanager
+def parquetfile(filepath: str):
+    def reader(pf: ParquetFile):
+        for batch in pf.iter_batches(PARQUET_BATCH_SIZE):
+            yield from batch.to_pylist()
+
+    with open(filepath, "rb") as fd:
+        yield reader(ParquetFile(fd))

{ingestr-0.13.3 → ingestr-0.13.5}/ingestr/src/sources.py

@@ -133,6 +133,11 @@ class SqlSource:
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
+        if uri.startswith("clickhouse://"):
+            uri = uri.replace("clickhouse://", "clickhouse+native://")
+            if "secure=" not in uri:
+                uri += "?secure=1"
+
         query_adapters = []
         if kwargs.get("sql_limit"):
             query_adapters.append(

ingestr-0.13.5/ingestr/src/version.py

@@ -0,0 +1,6 @@
+try:
+    from ingestr.src import buildinfo  # type: ignore
+
+    __version__ = buildinfo.version.lstrip("v")
+except ImportError:
+    __version__ = "0.0.0-dev"