omniload 0.0.0.dev0__tar.gz

omniload-0.0.0.dev0/.dlt/config.toml

@@ -0,0 +1,3 @@
+[runtime]
+
+dlthub_telemetry=false

omniload-0.0.0.dev0/.dockerignore

@@ -0,0 +1,137 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache/
+.dist/
+venv/
+
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+Dockerfile
+requirements-dev.txt

omniload-0.0.0.dev0/.githooks/pre-commit-hook.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+echo "scanning for secrets ..."
+
+WORK_DIR="/root/code"
+
+secret_detected() {
+    echo "secrets detected in source code. commit aborted."
+    exit 1
+}
+
+# use gitleaks binary if available
+# else fallback to using docker for running gitleaks
+CMD="gitleaks protect --staged -v"
+
+if [[ ! `which gitleaks`  ]]; then
+    which docker > /dev/null || (echo "gitleaks or docker is required for running secrets scan." && exit 1)
+    CMD="docker run -v $PWD:$WORK_DIR -w $WORK_DIR --rm ghcr.io/gitleaks/gitleaks:latest protect --staged -v"
+fi
+
+$CMD || secret_detected

omniload-0.0.0.dev0/.github/workflows/deploy-docs.yml

@@ -0,0 +1,62 @@
+name: Deploy VitePress site to Pages
+
+on:
+  push:
+    branches: [main]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Not needed if lastUpdated is not enabled
+      # - uses: pnpm/action-setup@v2 # Uncomment this if you're using pnpm
+      # - uses: oven-sh/setup-bun@v1 # Uncomment this if you're using Bun
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22.14.0'
+          cache: npm # or pnpm / yarn
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Install dependencies with npm@11 (for min-release-age support)
+        run: npx npm@11.10.0 ci
+      - name: Build with VitePress
+        run: |
+          npm run docs:build
+          touch docs/.vitepress/dist/.nojekyll
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: docs/.vitepress/dist
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    needs: build
+    runs-on: ubuntu-latest
+    name: Deploy
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

omniload-0.0.0.dev0/.github/workflows/release.yml

@@ -0,0 +1,149 @@
+# WARN: this is used as a notification filter in our slack.
+# When updating this, make sure to update the workflow subscription as well.
+name: "Publish to pip and ghcr.io"
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/${{ github.repository }}
+
+jobs:
+  tests:
+    uses: ./.github/workflows/tests.yml
+  pip-release:
+    runs-on: ubuntu-latest
+    needs: tests
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+        
+        # github actions invoke a fresh shell
+        # for each step. This causes us to lose our
+        # venv context. So we run all our build commands
+        # in a single step to simplify our pipeline.
+      - name: build and release
+        run: |
+          pip install uv
+          uv venv
+          source .venv/bin/activate
+          uv pip install -r requirements-dev.txt
+          make build
+          make upload-release
+
+  docker-build:
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-24.04-arm
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        id: build
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          tags: ${{ env.IMAGE }}
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,push-by-digest=true,name-canonical=true,push=true
+      
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          # The following strips the 'sha256:' prefix from the digest using bash parameter expansion.
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+    
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  docker-publish:
+    permissions:
+      contents: read
+      packages: write
+
+    runs-on: ubuntu-latest
+    needs:
+      - docker-build
+      - tests
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-*
+          merge-multiple: true
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.IMAGE }}
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern={{raw}}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.IMAGE }}@sha256:%s ' *)
+
+

omniload-0.0.0.dev0/.github/workflows/secrets-scan.yml

@@ -0,0 +1,12 @@
+name: secrets_scan
+on: [pull_request, push, workflow_dispatch]
+jobs:
+  scan:
+    name: gitleaks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: scan for secrets (gitleaks)
+        run: docker run -v $PWD:/code -w /code ghcr.io/gitleaks/gitleaks:latest dir -v

omniload-0.0.0.dev0/.github/workflows/tests.yml

@@ -0,0 +1,94 @@
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+  workflow_dispatch: {}
+  workflow_call: {}
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      code-changes: ${{ steps.filter.outputs.code-changes }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            code-changes:
+              - '**'
+              - '!**/*.md'
+              - '!docs/**'
+
+  tests:
+    needs: check-changes
+    if: github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || needs.check-changes.outputs.code-changes == 'true' || startsWith(github.ref, 'refs/tags/')
+    strategy:
+      fail-fast: false
+      matrix:
+        # I tried running stuff on macOS but it was too slow and unreliable.
+        # I also tried windows runners but couldn't get Docker to work there, so I gave up.
+        os: [ubuntu-latest]
+        python-version: [
+          '3.10',
+          '3.11',
+          '3.12',
+          '3.13',
+        ]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: install Microsoft ODBC
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo apt-get update
+          sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+      - name: install Microsoft ODBC
+        if: matrix.os == 'macos-13'
+        run: |
+          brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release
+          brew update
+          HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18
+      - name: Install Docker on macOS
+        if: matrix.os == 'macos-13'
+        run: |
+          brew install docker
+          brew install docker-compose
+          brew install colima
+          colima start
+          # Wait for Docker daemon to be ready
+          while ! docker system info > /dev/null 2>&1; do sleep 1; done
+      - name: install Microsoft ODBC
+        if: matrix.os == 'windows-latest'
+        run: |
+          Invoke-WebRequest -Uri https://go.microsoft.com/fwlink/?linkid=2249006 -OutFile msodbcsql.msi
+          Start-Process -FilePath "msiexec.exe" -ArgumentList "/i msodbcsql.msi /qn /norestart IACCEPTMSODBCSQLLICENSETERMS=YES" -Wait
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Install pip dependencies
+        run: make deps-ci
+
+      - name: run tests (macOS)
+        if: matrix.os == 'macos-13'
+        run: make test-ci
+        env:
+          TESTCONTAINERS_DOCKER_SOCKET_OVERRIDE: /var/run/docker.sock
+          DOCKER_HOST: unix:///Users/runner/.colima/docker.sock
+      - name: run tests (other OS)
+        if: matrix.os != 'macos-13'
+        run: make test-ci
+      - name: check the formatting
+        run: make lint-ci

omniload-0.0.0.dev0/.gitignore

@@ -0,0 +1,29 @@
+.DS_Store
+__pycache__
+.env
+venv
+!.gitkeep
+.idea
+.vscode/*
+.metals/*
+.coverage.*
+.coverage
+.ruff_cache
+.pytest_cache
+.mypy_cache
+pipeline_data
+dist
+docs/.vitepress/dist
+docs/.vitepress/cache
+node_modules
+*.duckdb
+*.duckdb.wal
+*.db
+*.db.wal
+*.sql
+ingest.sh
+test.env
+*.log
+.claude
+helper_scripts
+tmp

omniload-0.0.0.dev0/.gitleaksignore

@@ -0,0 +1,7 @@
+omniload/src/telemetry/event.py:generic-api-key:17
+omniload/src/testdata/fakebqcredentials.json:private-key:5
+docs/supported-sources/shopify.md:generic-api-key:26
+docs/supported-sources/smartsheets.md:generic-api-key:38
+omniload/src/google_analytics/helpers_test.py:generic-api-key:45
+omniload/src/google_analytics/helpers_test.py:generic-api-key:51
+omniload/src/google_analytics/helpers_test.py:generic-api-key:58

omniload-0.0.0.dev0/.npmrc

@@ -0,0 +1 @@
+min-release-age=7

omniload-0.0.0.dev0/.vale.ini

@@ -0,0 +1,16 @@
+StylesPath = styles
+
+MinAlertLevel = suggestion
+
+Packages = Google
+
+Vocab = omniload
+
+
+[*]
+BasedOnStyles = Vale, Google, omniload
+Google.We = NO
+Google.Passive = NO
+Google.Will = NO
+Google.Latin = NO
+Google.Contractions = NO

omniload-0.0.0.dev0/Dockerfile

@@ -0,0 +1,61 @@
+FROM python:3.11-slim-trixie
+
+# Guidelines that have been followed.
+# - https://hynek.me/articles/docker-uv/
+
+# Install the `uv` package manager.
+# Security-conscious organizations should package/review uv themselves.
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+# - Tell uv to byte-compile packages for faster application startups.
+# - Silence uv complaining about not being able to use hard links.
+# - Prevent uv from accidentally downloading isolated Python builds.
+# - Install packages into the system Python environment.
+ENV \
+    UV_COMPILE_BYTECODE=1 \
+    UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_SYSTEM_PYTHON=1
+
+WORKDIR /app
+
+
+# Install all prerequisites.
+
+RUN \
+  export ACCEPT_EULA='Y' && \
+  # Install build dependencies
+  apt-get update && \
+  apt-get install -y curl gcc gpg libpq-dev build-essential unixodbc-dev g++ apt-transport-https git
+
+RUN \ 
+  # Install pyodbc db drivers for MSSQL and PostgreSQL
+  curl -sSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /usr/share/keyrings/microsoft-prod.gpg && \
+  curl -sSL https://packages.microsoft.com/config/debian/12/prod.list | tee /etc/apt/sources.list.d/mssql-release.list
+  
+RUN \
+  # install the rest of them
+  apt-get update && \
+  ACCEPT_EULA=Y apt-get install -y msodbcsql18 odbc-postgresql && \
+  # Update odbcinst.ini to make sure full path to driver is listed, and set CommLog to 0. i.e disables any communication logs to be written to files
+  sed 's/Driver=psql/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psql/;s/CommLog=1/CommLog=0/' /etc/odbcinst.ini > /tmp/temp.ini && \
+  mv -f /tmp/temp.ini /etc/odbcinst.ini
+
+
+# Install application.
+
+# Copy sources and activate platform-specific requirements file.
+COPY . /app
+RUN if [ "$(uname -m)" = "aarch64" ]; then \
+    cp /app/requirements_arm64.txt /app/requirements.txt; \
+fi
+
+# Generate version file
+RUN make write-build-info
+
+# Install all required packages and the application.
+RUN uv pip install --requirement requirements.txt pyodbc .
+
+
+# Ready.
+ENTRYPOINT ["omniload"]