ingestr 0.7.3__tar.gz → 0.7.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. {ingestr-0.7.3 → ingestr-0.7.4}/.github/workflows/tests.yml +26 -26
  2. ingestr-0.7.4/Dockerfile +37 -0
  3. {ingestr-0.7.3 → ingestr-0.7.4}/PKG-INFO +1 -2
  4. ingestr-0.7.4/ingestr/src/version.py +1 -0
  5. {ingestr-0.7.3 → ingestr-0.7.4}/requirements.txt +0 -1
  6. ingestr-0.7.3/Dockerfile +0 -45
  7. ingestr-0.7.3/ingestr/src/version.py +0 -1
  8. {ingestr-0.7.3 → ingestr-0.7.4}/.dockerignore +0 -0
  9. {ingestr-0.7.3 → ingestr-0.7.4}/.github/workflows/deploy-docs.yml +0 -0
  10. {ingestr-0.7.3 → ingestr-0.7.4}/.gitignore +0 -0
  11. {ingestr-0.7.3 → ingestr-0.7.4}/.python-version +0 -0
  12. {ingestr-0.7.3 → ingestr-0.7.4}/LICENSE.md +0 -0
  13. {ingestr-0.7.3 → ingestr-0.7.4}/Makefile +0 -0
  14. {ingestr-0.7.3 → ingestr-0.7.4}/README.md +0 -0
  15. {ingestr-0.7.3 → ingestr-0.7.4}/docs/.vitepress/config.mjs +0 -0
  16. {ingestr-0.7.3 → ingestr-0.7.4}/docs/.vitepress/theme/custom.css +0 -0
  17. {ingestr-0.7.3 → ingestr-0.7.4}/docs/.vitepress/theme/index.js +0 -0
  18. {ingestr-0.7.3 → ingestr-0.7.4}/docs/commands/example-uris.md +0 -0
  19. {ingestr-0.7.3 → ingestr-0.7.4}/docs/commands/ingest.md +0 -0
  20. {ingestr-0.7.3 → ingestr-0.7.4}/docs/getting-started/core-concepts.md +0 -0
  21. {ingestr-0.7.3 → ingestr-0.7.4}/docs/getting-started/incremental-loading.md +0 -0
  22. {ingestr-0.7.3 → ingestr-0.7.4}/docs/getting-started/quickstart.md +0 -0
  23. {ingestr-0.7.3 → ingestr-0.7.4}/docs/getting-started/telemetry.md +0 -0
  24. {ingestr-0.7.3 → ingestr-0.7.4}/docs/index.md +0 -0
  25. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/bigquery.md +0 -0
  26. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/csv.md +0 -0
  27. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/databricks.md +0 -0
  28. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/duckdb.md +0 -0
  29. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/gorgias.md +0 -0
  30. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/gsheets.md +0 -0
  31. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/mongodb.md +0 -0
  32. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/mssql.md +0 -0
  33. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/mysql.md +0 -0
  34. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/notion.md +0 -0
  35. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/oracle.md +0 -0
  36. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/overview.md +0 -0
  37. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/postgres.md +0 -0
  38. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/redshift.md +0 -0
  39. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/sap-hana.md +0 -0
  40. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/shopify.md +0 -0
  41. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/snowflake.md +0 -0
  42. {ingestr-0.7.3 → ingestr-0.7.4}/docs/supported-sources/sqlite.md +0 -0
  43. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/main.py +0 -0
  44. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/destinations.py +0 -0
  45. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/factory.py +0 -0
  46. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/google_sheets/README.md +0 -0
  47. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/google_sheets/__init__.py +0 -0
  48. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/google_sheets/helpers/__init__.py +0 -0
  49. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/google_sheets/helpers/api_calls.py +0 -0
  50. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/google_sheets/helpers/data_processing.py +0 -0
  51. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/gorgias/__init__.py +0 -0
  52. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/gorgias/helpers.py +0 -0
  53. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/mongodb/__init__.py +0 -0
  54. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/mongodb/helpers.py +0 -0
  55. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/notion/__init__.py +0 -0
  56. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/notion/helpers/__init__.py +0 -0
  57. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/notion/helpers/client.py +0 -0
  58. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/notion/helpers/database.py +0 -0
  59. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/notion/settings.py +0 -0
  60. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/shopify/__init__.py +0 -0
  61. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/shopify/exceptions.py +0 -0
  62. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/shopify/helpers.py +0 -0
  63. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/shopify/settings.py +0 -0
  64. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sources.py +0 -0
  65. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sql_database/__init__.py +0 -0
  66. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sql_database/arrow_helpers.py +0 -0
  67. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sql_database/helpers.py +0 -0
  68. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sql_database/override.py +0 -0
  69. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/sql_database/schema_types.py +0 -0
  70. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/table_definition.py +0 -0
  71. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/telemetry/event.py +0 -0
  72. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/src/testdata/fakebqcredentials.json +0 -0
  73. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/.gitignore +0 -0
  74. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/create_replace.csv +0 -0
  75. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/delete_insert_expected.csv +0 -0
  76. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/delete_insert_part1.csv +0 -0
  77. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/delete_insert_part2.csv +0 -0
  78. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/merge_expected.csv +0 -0
  79. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/merge_part1.csv +0 -0
  80. {ingestr-0.7.3 → ingestr-0.7.4}/ingestr/testdata/merge_part2.csv +0 -0
  81. {ingestr-0.7.3 → ingestr-0.7.4}/package-lock.json +0 -0
  82. {ingestr-0.7.3 → ingestr-0.7.4}/package.json +0 -0
  83. {ingestr-0.7.3 → ingestr-0.7.4}/pyproject.toml +0 -0
  84. {ingestr-0.7.3 → ingestr-0.7.4}/requirements-dev.txt +0 -0
  85. {ingestr-0.7.3 → ingestr-0.7.4}/resources/demo.gif +0 -0
  86. {ingestr-0.7.3 → ingestr-0.7.4}/resources/demo.tape +0 -0
  87. {ingestr-0.7.3 → ingestr-0.7.4}/resources/ingestr.svg +0 -0
@@ -19,33 +19,33 @@ env:
19
19
 
20
20
 
21
21
  jobs:
22
- tests:
23
- runs-on: ubuntu-latest
24
- steps:
25
- - uses: actions/checkout@v4
26
- - name: install Microsoft ODBC
27
- run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
28
- - uses: actions/setup-python@v4
29
- with:
30
- python-version: '3.11'
31
- cache: 'pip'
32
- - name: Cache dependencies
33
- uses: actions/cache@v3
34
- id: cache
35
- with:
36
- path: ${{ env.pythonLocation }}
37
- key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
38
- - name: Install pip dependencies
39
- if: steps.cache.outputs.cache-hit != 'true'
40
- run: make deps-ci
41
- - name: run tests
42
- run: make test-ci
43
- - name: check the formatting
44
- run: make lint-ci
22
+ # tests:
23
+ # runs-on: ubuntu-latest
24
+ # steps:
25
+ # - uses: actions/checkout@v4
26
+ # - name: install Microsoft ODBC
27
+ # run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
28
+ # - uses: actions/setup-python@v4
29
+ # with:
30
+ # python-version: '3.11'
31
+ # cache: 'pip'
32
+ # - name: Cache dependencies
33
+ # uses: actions/cache@v3
34
+ # id: cache
35
+ # with:
36
+ # path: ${{ env.pythonLocation }}
37
+ # key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}
38
+ # - name: Install pip dependencies
39
+ # if: steps.cache.outputs.cache-hit != 'true'
40
+ # run: make deps-ci
41
+ # - name: run tests
42
+ # run: make test-ci
43
+ # - name: check the formatting
44
+ # run: make lint-ci
45
45
 
46
46
  build-and-push-image:
47
- needs: tests
48
- if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
47
+ # needs: tests
48
+ # if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
49
49
  runs-on: ubuntu-latest
50
50
  permissions:
51
51
  contents: read
@@ -72,7 +72,7 @@ jobs:
72
72
  uses: docker/build-push-action@v5
73
73
  with:
74
74
  context: .
75
- platforms: linux/amd64
75
+ platforms: linux/amd64,linux/arm64,linux/arm64/v8
76
76
  push: true
77
77
  tags: ${{ steps.meta.outputs.tags }}
78
78
  labels: ${{ steps.meta.outputs.labels }}
@@ -0,0 +1,37 @@
1
+ FROM python:3.11-slim
2
+
3
+ WORKDIR /app
4
+
5
+ COPY ./requirements.txt /app/requirements.txt
6
+
7
+ # Setup dependencies for pyodbc
8
+ RUN \
9
+ export ACCEPT_EULA='Y' && \
10
+ # Install build dependencies
11
+ apt-get update && \
12
+ apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https
13
+
14
+ RUN \
15
+ # Install pyodbc db drivers for MSSQL and PostgreSQL
16
+ curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
17
+ curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
18
+
19
+ RUN \
20
+ # install the rest of them
21
+ apt-get update && \
22
+ ACCEPT_EULA=Y apt-get install -y msodbcsql18 odbc-postgresql && \
23
+ # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
24
+ sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
25
+ mv -f /tmp/temp.ini /etc/odbcinst.ini
26
+
27
+ ENV VIRTUAL_ENV=/usr/local
28
+ ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
29
+ RUN /install.sh && rm /install.sh
30
+
31
+ RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
32
+
33
+ COPY . /app
34
+
35
+ RUN pip3 install -e .
36
+
37
+ ENTRYPOINT ["ingestr"]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: ingestr
3
- Version: 0.7.3
3
+ Version: 0.7.4
4
4
  Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
5
5
  Project-URL: Homepage, https://github.com/bruin-data/ingestr
6
6
  Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -14,7 +14,6 @@ Classifier: Operating System :: OS Independent
14
14
  Classifier: Programming Language :: Python :: 3
15
15
  Classifier: Topic :: Database
16
16
  Requires-Python: >=3.9
17
- Requires-Dist: connectorx==0.3.3
18
17
  Requires-Dist: cx-oracle==8.3.0
19
18
  Requires-Dist: databricks-sql-connector==2.9.3
20
19
  Requires-Dist: dlt==0.5.1
@@ -0,0 +1 @@
1
+ __version__ = "0.7.4"
@@ -1,4 +1,3 @@
1
- connectorx==0.3.3
2
1
  cx_Oracle==8.3.0
3
2
  databricks-sql-connector==2.9.3
4
3
  dlt==0.5.1
ingestr-0.7.3/Dockerfile DELETED
@@ -1,45 +0,0 @@
1
- FROM python:3.11-slim
2
-
3
- WORKDIR /app
4
-
5
- COPY ./requirements.txt /app/requirements.txt
6
-
7
- # Setup dependencies for pyodbc
8
- RUN \
9
- export ACCEPT_EULA='Y' && \
10
- export MYSQL_CONNECTOR='mysql-connector-odbc-8.0.33-linux-glibc2.28-x86-64bit' && \
11
- export MYSQL_CONNECTOR_CHECKSUM='41d03d5df0c631f8071cc697f7714620' && \
12
- # Install build dependencies
13
- apt-get update && \
14
- apt-get install -y curl gcc libpq-dev build-essential unixodbc-dev g++ apt-transport-https && \
15
- # Install pyodbc db drivers for MSSQL, PG and MySQL
16
- curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
17
- curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list && \
18
- # install the mysql connector
19
- curl -L -o ${MYSQL_CONNECTOR}.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/8.0/${MYSQL_CONNECTOR}.tar.gz && \
20
- echo "${MYSQL_CONNECTOR_CHECKSUM} ${MYSQL_CONNECTOR}.tar.gz" | md5sum -c - && \
21
- gunzip ${MYSQL_CONNECTOR}.tar.gz && tar xvf ${MYSQL_CONNECTOR}.tar && \
22
- cp -r ${MYSQL_CONNECTOR}/bin/* /usr/local/bin && cp -r ${MYSQL_CONNECTOR}/lib/* /usr/local/lib && \
23
- myodbc-installer -a -d -n "MySQL ODBC 8.0.33 Driver" -t "Driver=/usr/local/lib/libmyodbc8w.so" && \
24
- myodbc-installer -a -d -n "MySQL ODBC 8.0.33" -t "Driver=/usr/local/lib/libmyodbc8a.so" && \
25
- # install the rest of them
26
- apt-get update && \
27
- ACCEPT_EULA=Y apt-get install -y msodbcsql17 msodbcsql18 odbc-postgresql && \
28
- # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
29
- sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
30
- mv -f /tmp/temp.ini /etc/odbcinst.ini && \
31
- # Cleanup build dependencies
32
- rm -rf ${MYSQL_CONNECTOR}*
33
-
34
-
35
- ENV VIRTUAL_ENV=/usr/local
36
- ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh
37
- RUN /install.sh && rm /install.sh
38
-
39
- RUN /root/.cargo/bin/uv pip install --system --no-cache -r requirements.txt
40
-
41
- COPY . /app
42
-
43
- RUN pip3 install -e .
44
-
45
- ENTRYPOINT ["ingestr"]
@@ -1 +0,0 @@
1
- __version__ = "0.7.3"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes