ingestr 0.7.1__tar.gz → 0.7.3__tar.gz

ingestr-0.7.1/.github/workflows/docker.yml → ingestr-0.7.3/.github/workflows/tests.yml

@@ -5,39 +5,43 @@ on:
     tags:
       - "*"
   workflow_dispatch: {}
+  pull_request:
+    branches:
+      - main
 
 concurrency:
-  group: docker-${{ github.ref }}
-  cancel-in-progress: false
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
+
 jobs:
   tests:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+      - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
+          cache: 'pip'
       - name: Cache dependencies
         uses: actions/cache@v3
         id: cache
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
-      - name: install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+      - name: Install pip dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make deps-ci
       - name: run tests
-        run: make test
+        run: make test-ci
       - name: check the formatting
-        run: make lint
+        run: make lint-ci
 
   build-and-push-image:
     needs: tests
@@ -68,7 +72,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64,linux/arm64,linux/arm64/v8
+          platforms: linux/amd64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

ingestr-0.7.3/Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+
+# Setup dependencies for pyodbc
+RUN \
+  export ACCEPT_EULA='Y' && \
+  export MYSQL_CONNECTOR='mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit' && \
+  export MYSQL_CONNECTOR_CHECKSUM='41d03d5df0c631f8071cc697f7714620' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https && \
+  # Install pyodbc db drivers for MSSQL, PG and MySQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list && \
+  # install the mysql connector
+  curl -L -o ${MYSQL_CONNECTOR}.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/${MYSQL_CONNECTOR}.tar.gz && \
+  echo "${MYSQL_CONNECTOR_CHECKSUM} ${MYSQL_CONNECTOR}.tar.gz" | md5sum -c - && \
+  gunzip ${MYSQL_CONNECTOR}.tar.gz && tar xvf ${MYSQL_CONNECTOR}.tar && \
+  cp -r ${MYSQL_CONNECTOR}/bin/* /usr/local/bin && cp -r ${MYSQL_CONNECTOR}/lib/* /usr/local/lib && \
+  myodbc-installer -a -d -n "MySQL ODBC 8.0.33 Driver" -t "Driver=/usr/local/lib/libmyodbc8w.so" && \
+  myodbc-installer -a -d -n "MySQL ODBC 8.0.33" -t "Driver=/usr/local/lib/libmyodbc8a.so" && \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql17 msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini && \
+  # Cleanup build dependencies
+  rm -rf ${MYSQL_CONNECTOR}*
+
+
+ENV VIRTUAL_ENV=/usr/local
+ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
+RUN /install.sh && rm /install.sh
+
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
+
+COPY . /app
+
+RUN pip3 install -e .
+
+ENTRYPOINT ["ingestr"]

{ingestr-0.7.1 → ingestr-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.1
+Version: 0.7.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.7.1 → ingestr-0.7.3}/ingestr/main.py

@@ -244,8 +244,6 @@ def ingest(
         },
     )
 
-    dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True
-    dlt.config["normalize.parquet_normalizer.add_dlt_id"] = True
     dlt.config["data_writer.file_max_items"] = loader_file_size
     if schema_naming != SchemaNaming.default:
         dlt.config["schema.naming"] = schema_naming.value
@@ -280,7 +278,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector()
+            progressInstance = LogCollector(dump_system_stats=False)
 
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),

ingestr-0.7.3/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.7.3"

ingestr-0.7.1/.github/workflows/tests.yml

@@ -1,36 +0,0 @@
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: install Microsoft ODBC
-        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
-      - name: Install pip dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make deps-ci
-      - name: run tests
-        run: make test-ci
-      - name: check the formatting
-        run: make lint-ci

ingestr-0.7.1/Dockerfile

@@ -1,19 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-COPY ./requirements.txt /app/requirements.txt
-
-RUN apt-get update && apt-get -y install libpq-dev gcc curl g++
-
-ENV VIRTUAL_ENV=/usr/local
-ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
-RUN /install.sh && rm /install.sh
-
-RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
-
-COPY . /app
-
-RUN pip3 install -e .
-
-ENTRYPOINT ["ingestr"]

ingestr-0.7.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.7.1"