ingestr 0.7.0__tar.gz → 0.7.3__tar.gz

ingestr-0.7.0/.github/workflows/docker.yml → ingestr-0.7.3/.github/workflows/tests.yml

@@ -5,39 +5,43 @@ on:
     tags:
       - "*"
   workflow_dispatch: {}
+  pull_request:
+    branches:
+      - main
 
 concurrency:
-  group: docker-${{ github.ref }}
-  cancel-in-progress: false
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
+
 jobs:
   tests:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+      - name: install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+      - uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
+          cache: 'pip'
       - name: Cache dependencies
         uses: actions/cache@v3
         id: cache
         with:
           path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
-      - name: install uv
-        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
+      - name: Install pip dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: make deps-ci
       - name: run tests
-        run: make test
+        run: make test-ci
       - name: check the formatting
-        run: make lint
+        run: make lint-ci
 
   build-and-push-image:
     needs: tests
@@ -68,7 +72,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/amd64,linux/arm64,linux/arm64/v8
+          platforms: linux/amd64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

ingestr-0.7.3/Dockerfile

@@ -0,0 +1,45 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+
+# Setup dependencies for pyodbc
+RUN \
+  export ACCEPT_EULA='Y' && \
+  export MYSQL_CONNECTOR='mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit' && \
+  export MYSQL_CONNECTOR_CHECKSUM='41d03d5df0c631f8071cc697f7714620' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https && \
+  # Install pyodbc db drivers for MSSQL, PG and MySQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list && \
+  # install the mysql connector
+  curl -L -o ${MYSQL_CONNECTOR}.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/${MYSQL_CONNECTOR}.tar.gz && \
+  echo "${MYSQL_CONNECTOR_CHECKSUM} ${MYSQL_CONNECTOR}.tar.gz" | md5sum -c - && \
+  gunzip ${MYSQL_CONNECTOR}.tar.gz && tar xvf ${MYSQL_CONNECTOR}.tar && \
+  cp -r ${MYSQL_CONNECTOR}/bin/* /usr/local/bin && cp -r ${MYSQL_CONNECTOR}/lib/* /usr/local/lib && \
+  myodbc-installer -a -d -n "MySQL ODBC 8.0.33 Driver" -t "Driver=/usr/local/lib/libmyodbc8w.so" && \
+  myodbc-installer -a -d -n "MySQL ODBC 8.0.33" -t "Driver=/usr/local/lib/libmyodbc8a.so" && \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql17 msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini && \
+  # Cleanup build dependencies
+  rm -rf ${MYSQL_CONNECTOR}*
+
+
+ENV VIRTUAL_ENV=/usr/local
+ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
+RUN /install.sh && rm /install.sh
+
+RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
+
+COPY . /app
+
+RUN pip3 install -e .
+
+ENTRYPOINT ["ingestr"]

{ingestr-0.7.0 → ingestr-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ingestr
-Version: 0.7.0
+Version: 0.7.3
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,6 +14,7 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Database
 Requires-Python: >=3.9
+Requires-Dist: connectorx==0.3.3
 Requires-Dist: cx-oracle==8.3.0
 Requires-Dist: databricks-sql-connector==2.9.3
 Requires-Dist: dlt==0.5.1

{ingestr-0.7.0 → ingestr-0.7.3}/ingestr/main.py

@@ -104,6 +104,7 @@ class LoaderFileFormat(str, Enum):
 class SqlBackend(str, Enum):
     sqlalchemy = "sqlalchemy"
     pyarrow = "pyarrow"
+    connectorx = "connectorx"
 
 
 class Progress(str, Enum):
@@ -111,6 +112,11 @@ class Progress(str, Enum):
     log = "log"
 
 
+class SchemaNaming(str, Enum):
+    default = "default"
+    direct = "direct"
+
+
 @app.command()
 def ingest(
     source_uri: Annotated[
@@ -223,6 +229,13 @@ def ingest(
             envvar="LOADER_FILE_SIZE",
         ),
     ] = 100000,  # type: ignore
+    schema_naming: Annotated[
+        SchemaNaming,
+        typer.Option(
+            help="The naming convention to use when moving the tables from source to destination. The default behavior is explained here: https://dlthub.com/docs/general-usage/schema#naming-convention",
+            envvar="SCHEMA_NAMING",
+        ),
+    ] = SchemaNaming.default,  # type: ignore
 ):
     track(
         "command_triggered",
@@ -231,9 +244,9 @@ def ingest(
         },
     )
 
-    dlt.config["normalize.parquet_normalizer.add_dlt_load_id"] = True
-    dlt.config["normalize.parquet_normalizer.add_dlt_id"] = True
     dlt.config["data_writer.file_max_items"] = loader_file_size
+    if schema_naming != SchemaNaming.default:
+        dlt.config["schema.naming"] = schema_naming.value
 
     try:
         if not dest_table:
@@ -265,7 +278,7 @@ def ingest(
 
         progressInstance: Collector = SpinnerCollector()
         if progress == Progress.log:
-            progressInstance = LogCollector()
+            progressInstance = LogCollector(dump_system_stats=False)
 
         pipeline = dlt.pipeline(
             pipeline_name=m.hexdigest(),

ingestr-0.7.3/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.7.3"

{ingestr-0.7.0 → ingestr-0.7.3}/requirements.txt

@@ -1,10 +1,12 @@
+connectorx==0.3.3
 cx_Oracle==8.3.0
 databricks-sql-connector==2.9.3
 dlt==0.5.1
 duckdb_engine==0.11.5
 duckdb==0.10.2
-google-cloud-bigquery-storage==2.24.0
 google-api-python-client==2.130.0
+google-cloud-bigquery-storage==2.24.0
+mysql-connector-python==9.0.0
 pendulum==3.0.0
 psycopg2-binary==2.9.9
 py-machineid==0.5.1
@@ -17,10 +19,9 @@ rich==13.7.1
 rudder-sdk-python==2.1.0
 snowflake-sqlalchemy==1.5.3
 sqlalchemy-bigquery==1.11.0
+sqlalchemy-hana==2.0.0
 sqlalchemy-redshift==0.8.14
 SQLAlchemy==1.4.52
 sqlalchemy2-stubs==0.0.2a38
 tqdm==4.66.2
 typer==0.12.3
-sqlalchemy-hana==2.0.0
-mysql-connector-python==9.0.0

ingestr-0.7.0/.github/workflows/tests.yml

@@ -1,36 +0,0 @@
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  tests:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: install Microsoft ODBC
-        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
-      - name: Install pip dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make deps-ci
-      - name: run tests
-        run: make test-ci
-      - name: check the formatting
-        run: make lint-ci

ingestr-0.7.0/Dockerfile

@@ -1,19 +0,0 @@
-FROM python:3.11-slim
-
-WORKDIR /app
-
-COPY ./requirements.txt /app/requirements.txt
-
-RUN apt-get update && apt-get -y install libpq-dev gcc curl g++
-
-ENV VIRTUAL_ENV=/usr/local
-ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
-RUN /install.sh && rm /install.sh
-
-RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
-
-COPY . /app
-
-RUN pip3 install -e .
-
-ENTRYPOINT ["ingestr"]

ingestr-0.7.0/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.7.0"