ingestr 0.13.49__tar.gz → 0.14.98__tar.gz

ingestr-0.14.98/.dlt/config.toml

@@ -0,0 +1,3 @@
+[runtime]
+
+dlthub_telemetry=false

ingestr-0.14.98/.github/workflows/release.yml

@@ -0,0 +1,151 @@
+# WARN: this is used as a notification filter in our slack.
+# When updating this, make sure to update the workflow subscription as well.
+name: "Release ingestr on pip and ghcr.io"
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/${{ github.repository }}
+
+jobs:
+  tests:
+    uses: ./.github/workflows/tests.yml
+  pip-release:
+    runs-on: ubuntu-latest
+    needs: tests
+    env:
+      TWINE_PASSWORD: ${{ secrets.PYPI_INGESTR_TOKEN }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+        
+        # github actions invoke a fresh shell
+        # for each step. This causes us to lose our
+        # venv context. So we run all our build commands
+        # in a single step to simplify our pipeline.
+      - name: build and release
+        run: |
+          pip install uv
+          uv venv
+          source .venv/bin/activate
+          uv pip install -r requirements-dev.txt
+          make build
+          make upload-release
+
+  docker-build:
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-24.04-arm
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        id: build
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          tags: ${{ env.IMAGE }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
+      
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          # The following strips the 'sha256:' prefix from the digest using bash parameter expansion.
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+    
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  docker-publish:
+    permissions:
+      contents: read
+      packages: write
+
+    runs-on: ubuntu-latest
+    needs:
+      - docker-build
+      - tests
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern={{raw}}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
+
+

{ingestr-0.13.49 → ingestr-0.14.98}/.github/workflows/tests.yml

@@ -13,13 +13,35 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      code-changes: ${{ steps.filter.outputs.code-changes }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            code-changes:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**'
+
   tests:
+    needs: check-changes
+    if: needs.check-changes.outputs.code-changes == 'true'
     strategy:
       matrix:
         # I tried running stuff on macOS but it was too slow and unreliable.
         # I also tried windows runners but couldn't get Docker to work there, so I gave up.
         os: [ubuntu-latest]
-        python-version: ['3.10', '3.11', '3.12']
+        python-version: [
+          '3.10',
+          '3.11',
+          '3.12',
+          '3.13',
+        ]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -53,6 +75,7 @@ jobs:
         uses: astral-sh/setup-uv@v3
       - name: Install pip dependencies
         run: make deps-ci
+
       - name: run tests (macOS)
         if: matrix.os == 'macos-13'
         run: make test-ci

{ingestr-0.13.49 → ingestr-0.14.98}/.gitignore

@@ -3,6 +3,7 @@ __pycache__
 .env
 venv
 !.gitkeep
+.idea
 .vscode/*
 .metals/*
 .coverage.*
@@ -22,4 +23,6 @@ node_modules
 *.sql
 ingest.sh
 test.env
-*.log
+*.log
+.claude
+helper_scripts

ingestr-0.14.98/Dockerfile

@@ -0,0 +1,61 @@
+FROM python:3.11-slim-trixie
+
+# Guidelines that have been followed.
+# - https://hynek.me/articles/docker-uv/
+
+# Install the `uv` package manager.
+# Security-conscious organizations should package/review uv themselves.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+# - Tell uv to byte-compile packages for faster application startups.
+# - Silence uv complaining about not being able to use hard links.
+# - Prevent uv from accidentally downloading isolated Python builds.
+# - Install packages into the system Python environment.
+ENV \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_SYSTEM_PYTHON=1
+
+WORKDIR /app
+
+
+# Install all prerequisites.
+
+RUN \
+  export ACCEPT_EULA='Y' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc gpg libpq-dev build-essential unixodbc-dev g++ apt-transport-https git
+
+RUN \ 
+  # Install pyodbc db drivers for MSSQL and PostgreSQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
+  
+RUN \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini
+
+
+# Install application.
+
+# Copy sources and activate platform-specific requirements file.
+COPY . /app
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    cp /app/requirements_arm64.txt /app/requirements.txt; \
+fi
+
+# Generate version file
+RUN make write-build-info
+
+# Install all required packages and the application.
+RUN uv pip install --requirement requirements.txt pyodbc .
+
+
+# Ready.
+ENTRYPOINT ["ingestr"]

{ingestr-0.13.49 → ingestr-0.14.98}/Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 .ONESHELL:
-.PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell
+.PHONY: test lint format test-ci lint-ci build upload-release setup docker-shell write-build-info remove-build-info
 
 BUILDINFO=ingestr/src/buildinfo.py
 
@@ -31,11 +31,14 @@ test-specific: venv lock-deps
 	. venv/bin/activate; set -a; source test.env; set +a; TESTCONTAINERS_RYUK_DISABLED=true pytest -n auto  -rP -vv --tb=short --capture=no -k $(test)
 
 lint-ci:
-	ruff format ingestr && ruff check ingestr --fix
-	mypy --config-file pyproject.toml --explicit-package-bases ingestr
+	@ruff format --diff ingestr && ruff check ingestr && mypy --config-file pyproject.toml --explicit-package-bases ingestr
+
+format:
+	@ruff format ingestr && ruff check ingestr --fix
+	@mypy --config-file pyproject.toml --explicit-package-bases ingestr
 
 lint: venv lock-deps
-	. venv/bin/activate; $(MAKE) lint-ci
+	. venv/bin/activate; $(MAKE) format
 
 lint-docs:
 	vale docs --glob='!**/.vitepress/**'
@@ -43,8 +46,14 @@ lint-docs:
 tl: test lint
 
 build: lock-deps
-	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
+	$(MAKE) write-build-info
 	rm -rf dist && python3 -m build
+	$(MAKE) remove-build-info
+
+write-build-info:
+	cat > ${BUILDINFO} <<< "version = \"$$(git describe --tags --abbrev=0)\""
+
+remove-build-info:
 	rm -f ${BUILDINFO}
 
 upload-release:

{ingestr-0.13.49 → ingestr-0.14.98}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.49
+Version: 0.14.98
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -26,7 +26,7 @@ Requires-Dist: asn1crypto==1.5.1
 Requires-Dist: asynch==0.2.4
 Requires-Dist: attrs==25.1.0
 Requires-Dist: backoff==2.2.1
-Requires-Dist: beautifulsoup4==4.13.3
+Requires-Dist: bcrypt==4.3.0
 Requires-Dist: boto3==1.37.1
 Requires-Dist: botocore==1.37.1
 Requires-Dist: cachetools==5.5.2
@@ -39,26 +39,33 @@ Requires-Dist: clickhouse-connect==0.8.14
 Requires-Dist: clickhouse-driver==0.2.9
 Requires-Dist: clickhouse-sqlalchemy==0.2.7
 Requires-Dist: confluent-kafka==2.8.0
+Requires-Dist: couchbase==4.3.6
+Requires-Dist: crate==2.0.0
 Requires-Dist: cryptography==44.0.2
 Requires-Dist: curlify==2.2.1
-Requires-Dist: databricks-sql-connector==2.9.3
+Requires-Dist: databricks-sql-connector==4.0.5
 Requires-Dist: databricks-sqlalchemy==1.0.2
 Requires-Dist: dataclasses-json==0.6.7
 Requires-Dist: decorator==5.2.1
 Requires-Dist: deprecation==2.1.0
-Requires-Dist: dlt==1.10.0
+Requires-Dist: dlt-cratedb==0.0.2
+Requires-Dist: dlt==1.16.0
 Requires-Dist: dnspython==2.7.0
 Requires-Dist: duckdb-engine==0.17.0
 Requires-Dist: duckdb==1.2.1
+Requires-Dist: ecdsa==0.19.1
 Requires-Dist: elastic-transport==8.17.1
 Requires-Dist: elasticsearch==8.10.1
+Requires-Dist: enum-compat==0.0.3
 Requires-Dist: et-xmlfile==2.0.0
-Requires-Dist: facebook-business==20.0.0
+Requires-Dist: facebook-business==23.0.0
 Requires-Dist: filelock==3.17.0
 Requires-Dist: flatten-json==0.1.14
 Requires-Dist: frozenlist==1.5.0
 Requires-Dist: fsspec==2025.3.2
+Requires-Dist: future==1.0.0
 Requires-Dist: gcsfs==2025.3.2
+Requires-Dist: geojson==3.2.0
 Requires-Dist: gitdb==4.0.12
 Requires-Dist: gitpython==3.1.44
 Requires-Dist: giturlparse==0.12.0
@@ -77,7 +84,7 @@ Requires-Dist: google-cloud-storage==3.1.0
 Requires-Dist: google-crc32c==1.6.0
 Requires-Dist: google-resumable-media==2.7.2
 Requires-Dist: googleapis-common-protos==1.69.0
-Requires-Dist: greenlet==3.2.2
+Requires-Dist: greenlet==3.2.4
 Requires-Dist: grpc-google-iam-v1==0.14.2
 Requires-Dist: grpc-interceptor==0.15.4
 Requires-Dist: grpcio-status==1.62.3
@@ -86,17 +93,16 @@ Requires-Dist: hdbcli==2.23.27
 Requires-Dist: hexbytes==1.3.0
 Requires-Dist: httplib2==0.22.0
 Requires-Dist: humanize==4.12.1
-Requires-Dist: ibm-db-sa==0.4.1
-Requires-Dist: ibm-db==3.2.6
 Requires-Dist: idna==3.10
 Requires-Dist: inflection==0.5.1
+Requires-Dist: influxdb-client==1.41.0
+Requires-Dist: intuit-oauth==1.2.4
 Requires-Dist: isodate==0.7.2
 Requires-Dist: jmespath==1.0.1
 Requires-Dist: jsonpath-ng==1.7.0
 Requires-Dist: leb128==1.0.8
 Requires-Dist: lxml==5.3.1
 Requires-Dist: lz4==4.4.3
-Requires-Dist: makefun==1.15.6
 Requires-Dist: mako==1.3.9
 Requires-Dist: markdown-it-py==3.0.0
 Requires-Dist: markupsafe==3.0.2
@@ -110,9 +116,10 @@ Requires-Dist: mysql-connector-python==9.2.0
 Requires-Dist: numpy==2.2.3
 Requires-Dist: oauthlib==3.2.2
 Requires-Dist: openpyxl==3.1.5
-Requires-Dist: orjson==3.10.15
+Requires-Dist: orjson==3.11.3
 Requires-Dist: packaging==24.2
 Requires-Dist: pandas==2.2.3
+Requires-Dist: paramiko==3.5.1
 Requires-Dist: pathvalidate==3.2.3
 Requires-Dist: pendulum==3.0.0
 Requires-Dist: platformdirs==4.3.6
@@ -137,18 +144,22 @@ Requires-Dist: pygments==2.19.1
 Requires-Dist: pyjwt==2.10.1
 Requires-Dist: pymongo==4.11.1
 Requires-Dist: pymysql==1.1.1
+Requires-Dist: pynacl==1.5.0
 Requires-Dist: pyopenssl==25.0.0
 Requires-Dist: pyparsing==3.2.1
 Requires-Dist: pyrate-limiter==3.7.0
 Requires-Dist: python-dateutil==2.9.0.post0
 Requires-Dist: python-dotenv==1.0.1
+Requires-Dist: python-jose==3.5.0
+Requires-Dist: python-quickbooks==0.9.2
 Requires-Dist: pytz==2025.1
 Requires-Dist: pyyaml==6.0.2
-Requires-Dist: redshift-connector==2.1.5
+Requires-Dist: rauth==0.7.3
+Requires-Dist: reactivex==4.0.4
 Requires-Dist: requests-file==2.1.0
 Requires-Dist: requests-oauthlib==1.3.1
 Requires-Dist: requests-toolbelt==1.0.0
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.5
 Requires-Dist: requirements-parser==0.11.0
 Requires-Dist: rich-argparse==1.7.0
 Requires-Dist: rich==13.9.4
@@ -156,7 +167,6 @@ Requires-Dist: rsa==4.9
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2025.3.2
 Requires-Dist: s3transfer==0.11.3
-Requires-Dist: scramp==1.4.5
 Requires-Dist: semver==3.0.4
 Requires-Dist: setuptools==75.8.2
 Requires-Dist: shellingham==1.5.4
@@ -168,8 +178,8 @@ Requires-Dist: smmap==5.0.2
 Requires-Dist: snowflake-connector-python==3.14.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
 Requires-Dist: sortedcontainers==2.4.0
-Requires-Dist: soupsieve==2.6
 Requires-Dist: sqlalchemy-bigquery==1.12.1
+Requires-Dist: sqlalchemy-cratedb==0.41.0
 Requires-Dist: sqlalchemy-hana==2.0.0
 Requires-Dist: sqlalchemy-redshift==0.8.14
 Requires-Dist: sqlalchemy-spanner==1.11.0
@@ -183,6 +193,7 @@ Requires-Dist: thrift==0.16.0
 Requires-Dist: time-machine==2.16.0
 Requires-Dist: tomlkit==0.13.2
 Requires-Dist: tqdm==4.67.1
+Requires-Dist: trino==0.336.0
 Requires-Dist: typer==0.13.1
 Requires-Dist: types-requests==2.32.0.20240907
 Requires-Dist: types-setuptools==75.8.2.20250305
@@ -192,11 +203,15 @@ Requires-Dist: tzdata==2025.1
 Requires-Dist: tzlocal==5.3
 Requires-Dist: uritemplate==4.1.1
 Requires-Dist: urllib3==2.3.0
+Requires-Dist: verlib2==0.2.0
 Requires-Dist: wrapt==1.17.2
 Requires-Dist: yarl==1.18.3
 Requires-Dist: zeep==4.3.1
 Requires-Dist: zstandard==0.23.0
 Requires-Dist: zstd==1.5.6.5
+Provides-Extra: ibm-db
+Requires-Dist: ibm-db-sa==0.4.1; extra == 'ibm-db'
+Requires-Dist: ibm-db==3.2.6; extra == 'ibm-db'
 Provides-Extra: odbc
 Requires-Dist: pyodbc==5.1.0; extra == 'odbc'
 Provides-Extra: oracle
@@ -292,11 +307,21 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>CrateDB</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td>Databricks</td>
         <td>✅</td>
         <td>✅</td>
     </tr>
+    <tr>
+        <td>IBM Db2</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>DuckDB</td>
         <td>✅</td>
@@ -307,6 +332,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Elasticsearch</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
+     <tr>
+        <td>GCP Spanner</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Local CSV file</td>
         <td>✅</td>
@@ -320,7 +355,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
     <tr>
         <td>MongoDB</td>
         <td>✅</td>
-        <td>❌</td>
+        <td>✅</td>
+    </tr>
+    <tr>
+        <td>MotherDuck</td>
+        <td>✅</td>
+        <td>✅</td>
     </tr>
     <tr>
         <td>MySQL</td>
@@ -357,6 +397,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>❌</td>
     </tr>
+    <tr>
+        <td>Trino</td>
+        <td>✅</td>
+        <td>✅</td>
+    </tr>
     <tr>
         <td colspan="3" style='text-align:center;'><strong>Platforms</strong></td>
     </tr>
@@ -393,6 +438,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Attio</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Chess.com</td>
         <td>✅</td>
@@ -404,7 +454,7 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Github</td>
+        <td>GitHub</td>
         <td>✅</td>
         <td>-</td>
     </tr>
@@ -447,6 +497,16 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>Personio</td>
         <td>✅</td>
         <td>-</td>
+    </tr>
+     <tr>
+        <td>Phantombuster</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Pipedrive</td>
+        <td>✅</td>
+        <td>-</td>
     </tr>
      <tr>
         <td>S3</td>
@@ -469,7 +529,12 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>-</td>
     </tr>
     <tr>
-        <td>Smartsheet</td>
+        <td>Smartsheets</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
+    <tr>
+        <td>Solidgate</td>
         <td>✅</td>
         <td>-</td>
     </tr>