ingestr 0.7.1__tar.gz → 0.7.4__tar.gz

ingestr-0.7.1/.github/workflows/docker.yml → ingestr-0.7.4/.github/workflows/tests.yml

@@ -5,43 +5,47 @@ on:
     tags:
       - "*"
   workflow_dispatch: {}
+  pull_request:
+    branches:
+      - main
 
 concurrency:
-  group: docker-${{ github.ref }}
-  cancel-in-progress: false
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
+
 jobs:
-  tests:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
-      - name: install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
-      - name: run tests
-        run: make test
-      - name: check the formatting
-        run: make lint
+  # tests:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: install Microsoft ODBC
+  #       run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: '3.11'
+  #         cache: 'pip'
+  #     - name: Cache dependencies
+  #       uses: actions/cache@v3
+  #       id: cache
+  #       with:
+  #         path: ${{ env.pythonLocation }}
+  #         key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+  #     - name: Install pip dependencies
+  #       if: steps.cache.outputs.cache-hit != 'true'
+  #       run: make deps-ci
+  #     - name: run tests
+  #       run: make test-ci
+  #     - name: check the formatting
+  #       run: make lint-ci
 
   build-and-push-image:
-    needs: tests
-    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    # needs: tests
+    # if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     permissions:
       contents: read

ingestr-0.7.4/Dockerfile

@@ -0,0 +1,37 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+
+# Setup dependencies for pyodbc
+RUN \
+  export ACCEPT_EULA='Y' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https
+
+RUN \ 
+  # Install pyodbc db drivers for MSSQL and PostgreSQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
+  
+RUN \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini
+
+ENV VIRTUAL_ENV=/usr/local
+ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
+RUN /install.sh && rm /install.sh
+
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
+
+COPY . /app
+
+RUN pip3 install -e .
+
+ENTRYPOINT ["ingestr"]

{ingestr-0.7.1 → ingestr-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.1
+Version: 0.7.4
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,7 +14,6 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
-Requires-Dist: connectorx==0.3.3
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1

{ingestr-0.7.1 → ingestr-0.7.4}/ingestr/main.py

@@ -244,8 +244,6 @@ def ingest(
         },
     )
 
-    dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True
-    dlt.config["normalize.parquet_normalizer.add_dlt_id"] = True
     dlt.config["data_writer.file_max_items"] = loader_file_size
     if schema_naming != SchemaNaming.default:
         dlt.config["schema.naming"] = schema_naming.value
@@ -280,7 +278,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector()
+            progressInstance = LogCollector(dump_system_stats=False)
 
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),

ingestr-0.7.4/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.7.4"

{ingestr-0.7.1 → ingestr-0.7.4}/requirements.txt

@@ -1,4 +1,3 @@
-connectorx==0.3.3
 cx_Oracle==8.3.0
 databricks-sql-connector==2.9.3
 dlt==0.5.1

ingestr-0.7.1/.github/workflows/tests.yml

@@ -1,36 +0,0 @@
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: install Microsoft ODBC
-        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
-      - name: Install pip dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make deps-ci
-      - name: run tests
-        run: make test-ci
-      - name: check the formatting
-        run: make lint-ci

ingestr-0.7.1/Dockerfile

@@ -1,19 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-COPY ./requirements.txt /app/requirements.txt
-
-RUN apt-get update && apt-get -y install libpq-dev gcc curl g++
-
-ENV VIRTUAL_ENV=/usr/local
-ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
-RUN /install.sh && rm /install.sh
-
-RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
-
-COPY . /app
-
-RUN pip3 install -e .
-
-ENTRYPOINT ["ingestr"]

ingestr-0.7.1/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.7.1"