etlrelay 0.1.0__tar.gz

etlrelay-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - dev
+      - main
+  pull_request:
+    branches:
+      - dev
+      - main
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quality:
+    name: quality
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Install locked project and development dependencies
+        run: uv sync --locked --all-groups
+
+      - name: Run tests with branch coverage
+        run: >-
+          uv run pytest
+          --cov=etlrelay
+          --cov-branch
+          --cov-report=term-missing
+          --cov-report=xml
+          --cov-fail-under=90.1
+
+      - name: Check lint
+        run: uv run ruff check src tests
+
+      - name: Check formatting
+        run: uv run ruff format --check src tests
+
+      - name: Check types
+        run: uv run pyright
+
+      - name: Build documentation
+        run: uv run mkdocs build --strict
+
+      - name: Build source distribution and wheel
+        run: uv build
+
+      - name: Validate distribution metadata
+        run: uvx --from 'twine>=7,<8' twine check dist/*

etlrelay-0.1.0/.github/workflows/publish-testpypi.yml

@@ -0,0 +1,176 @@
+name: Publish to TestPyPI
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: verify and build
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.metadata.outputs.version }}
+      distribution: ${{ steps.metadata.outputs.distribution }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Require the dev branch
+        shell: bash
+        run: |
+          if [[ "$GITHUB_REF_NAME" != "dev" ]]; then
+            echo "TestPyPI publishing must be run from the dev branch; selected: $GITHUB_REF_NAME" >&2
+            exit 1
+          fi
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Read package metadata
+        id: metadata
+        shell: bash
+        run: |
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          import os
+          import re
+          import tomllib
+          from pathlib import Path
+
+          project = tomllib.loads(Path("pyproject.toml").read_text())["project"]
+          distribution = re.sub(r"[-_.]+", "-", project["name"]).lower()
+          if distribution != "etlrelay":
+              raise SystemExit(f"Expected distribution name 'etlrelay', got {project['name']!r}")
+          print(f"version={project['version']}")
+          print(f"distribution={distribution}")
+          PY
+
+      - name: Install locked project and development dependencies
+        run: uv sync --locked --all-groups
+
+      - name: Run tests with branch coverage
+        run: >-
+          uv run pytest
+          --cov=etlrelay
+          --cov-branch
+          --cov-report=term-missing
+          --cov-report=xml
+          --cov-fail-under=90.1
+
+      - name: Check lint
+        run: uv run ruff check src tests
+
+      - name: Check formatting
+        run: uv run ruff format --check src tests
+
+      - name: Check types
+        run: uv run pyright
+
+      - name: Build documentation
+        run: uv run mkdocs build --strict
+
+      - name: Build source distribution and wheel
+        run: uv build
+
+      - name: Validate distribution metadata
+        run: uvx --from 'twine>=7,<8' twine check dist/*
+
+      - name: Upload built distributions
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: testpypi-distributions
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    name: publish to TestPyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/project/etlrelay/${{ needs.build.outputs.version }}/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download built distributions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: testpypi-distributions
+          path: dist/
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+  smoke-test:
+    name: install and smoke-test TestPyPI package
+    needs: [build, publish]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out source for runtime dependency metadata
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Export runtime dependencies without the checkout package
+        run: >-
+          uv export
+          --no-dev
+          --no-default-groups
+          --no-emit-project
+          --format requirements.txt
+          --output-file /tmp/etlrelay-runtime-requirements.txt
+
+      - name: Create a clean environment and install runtime dependencies
+        run: |
+          uv venv --python 3.12 /tmp/etlrelay-smoke
+          uv pip install --python /tmp/etlrelay-smoke/bin/python \
+            -r /tmp/etlrelay-runtime-requirements.txt
+
+      - name: Wait for TestPyPI to index the release
+        env:
+          PACKAGE_VERSION: ${{ needs.build.outputs.version }}
+        shell: bash
+        run: |
+          for attempt in $(seq 1 24); do
+            if curl --fail --silent \
+              "https://test.pypi.org/pypi/etlrelay/${PACKAGE_VERSION}/json" \
+              >/dev/null; then
+              echo "TestPyPI has indexed etlrelay ${PACKAGE_VERSION}."
+              exit 0
+            fi
+            echo "Waiting for TestPyPI indexing (attempt ${attempt}/24)..."
+            sleep 5
+          done
+          echo "TestPyPI did not index etlrelay ${PACKAGE_VERSION} within two minutes." >&2
+          exit 1
+
+      - name: Install the published TestPyPI distribution
+        env:
+          PACKAGE_VERSION: ${{ needs.build.outputs.version }}
+        run: >-
+          uv pip install
+          --python /tmp/etlrelay-smoke/bin/python
+          --index-url https://test.pypi.org/simple/
+          --no-deps
+          "etlrelay==${PACKAGE_VERSION}"
+
+      - name: Smoke-test installed package outside the repository
+        run: |
+          cd /tmp
+          /tmp/etlrelay-smoke/bin/python -c \
+            "from etlrelay.core.batch import Batch; from etlrelay.pipeline.pipeline import Pipeline; print('ETLRelay installed-package imports succeeded')"

etlrelay-0.1.0/.github/workflows/release-pypi.yml

@@ -0,0 +1,242 @@
+name: Publish release to PyPI
+
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  verify-build:
+    # Keep false while bootstrapping TestPyPI; set the repository variable to true
+    # only after the TestPyPI candidate has passed its smoke test.
+    if: github.ref == 'refs/heads/main' && vars.PYPI_RELEASE_ENABLED == 'true'
+    name: verify and build release
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.metadata.outputs.version }}
+      distribution: ${{ steps.metadata.outputs.distribution }}
+      already_published: ${{ steps.pypi-check.outputs.already_published }}
+    steps:
+      - name: Check out repository and tags
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Read package metadata
+        id: metadata
+        shell: bash
+        run: |
+          python - <<'PY' >> "$GITHUB_OUTPUT"
+          import re
+          import tomllib
+          from pathlib import Path
+
+          project = tomllib.loads(Path("pyproject.toml").read_text())["project"]
+          distribution = re.sub(r"[-_.]+", "-", project["name"]).lower()
+          if distribution != "etlrelay":
+              raise SystemExit(f"Expected distribution name 'etlrelay', got {project['name']!r}")
+          print(f"version={project['version']}")
+          print(f"distribution={distribution}")
+          PY
+
+      - name: Check whether this exact version is already published
+        id: pypi-check
+        env:
+          PACKAGE_VERSION: ${{ steps.metadata.outputs.version }}
+          DISTRIBUTION: ${{ steps.metadata.outputs.distribution }}
+        shell: bash
+        run: |
+          status=$(curl --silent --show-error --output /dev/null \
+            --write-out '%{http_code}' \
+            "https://pypi.org/pypi/${DISTRIBUTION}/${PACKAGE_VERSION}/json")
+
+          if [[ "$status" == "200" ]]; then
+            echo "already_published=true" >> "$GITHUB_OUTPUT"
+            echo "${DISTRIBUTION} ${PACKAGE_VERSION} already exists on PyPI; publication will be skipped."
+            exit 0
+          fi
+
+          if [[ "$status" != "404" ]]; then
+            echo "Unexpected PyPI response HTTP ${status}; refusing to publish." >&2
+            exit 1
+          fi
+
+          if git ls-remote --exit-code --tags origin "refs/tags/v${PACKAGE_VERSION}" >/dev/null 2>&1; then
+            echo "Release tag v${PACKAGE_VERSION} already exists, but that version was not found on PyPI." >&2
+            echo "Investigate this inconsistent release state before retrying." >&2
+            exit 1
+          fi
+
+          echo "already_published=false" >> "$GITHUB_OUTPUT"
+          echo "Version ${PACKAGE_VERSION} is not on PyPI; running release checks."
+
+      - name: Install locked project and development dependencies
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv sync --locked --all-groups
+
+      - name: Run tests with branch coverage
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: >-
+          uv run pytest
+          --cov=etlrelay
+          --cov-branch
+          --cov-report=term-missing
+          --cov-report=xml
+          --cov-fail-under=90.1
+
+      - name: Check lint
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv run ruff check src tests
+
+      - name: Check formatting
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv run ruff format --check src tests
+
+      - name: Check types
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv run pyright
+
+      - name: Build documentation
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv run mkdocs build --strict
+
+      - name: Build source distribution and wheel
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uv build
+
+      - name: Validate distribution metadata
+        if: steps.pypi-check.outputs.already_published == 'false'
+        run: uvx --from 'twine>=7,<8' twine check dist/*
+
+      - name: Upload built distributions
+        if: steps.pypi-check.outputs.already_published == 'false'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: pypi-distributions
+          path: dist/*
+          if-no-files-found: error
+          retention-days: 7
+
+  publish:
+    name: publish to PyPI
+    needs: verify-build
+    if: needs.verify-build.outputs.already_published == 'false'
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/etlrelay/${{ needs.verify-build.outputs.version }}/
+    permissions:
+      id-token: write
+    steps:
+      - name: Download verified distributions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: pypi-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  publish-versioned-docs:
+    name: publish versioned documentation
+    needs: [verify-build, publish]
+    # Only make a docs snapshot after a new package version has actually reached PyPI.
+    if: ${{ needs.verify-build.outputs.already_published == 'false' && needs.publish.result == 'success' }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Check out the released source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
+        with:
+          enable-cache: true
+
+      - name: Install Python
+        run: uv python install 3.12
+
+      - name: Install documentation and project dependencies
+        run: uv sync --locked --all-groups
+
+      - name: Configure Git identity for mike
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Publish the version and update the latest alias
+        env:
+          RELEASE_VERSION: ${{ needs.verify-build.outputs.version }}
+        run: |
+          uv run mike deploy --push --update-aliases "$RELEASE_VERSION" latest
+          uv run mike set-default --push latest
+
+  deploy-versioned-docs-to-github-pages:
+    name: deploy documentation to GitHub Pages
+    needs: publish-versioned-docs
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Configure GitHub Pages
+        uses: actions/configure-pages@v5
+
+      - name: Check out the mike-generated site
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: gh-pages
+          path: gh-pages
+          persist-credentials: false
+
+      - name: Prepare Pages artifact without Git metadata
+        run: |
+          mkdir site-artifact
+          rsync -a --exclude='.git' gh-pages/ site-artifact/
+
+      - name: Upload Pages artifact
+        uses: actions/upload-pages-artifact@v4
+        with:
+          path: site-artifact/
+
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+
+  create-github-release:
+    name: create GitHub release
+    needs: [verify-build, publish]
+    if: needs.verify-build.outputs.already_published == 'false' && needs.publish.result == 'success'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Create tag and GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PACKAGE_VERSION: ${{ needs.verify-build.outputs.version }}
+        run: |
+          gh release create "v${PACKAGE_VERSION}" \
+            --repo "$GITHUB_REPOSITORY" \
+            --target "$GITHUB_SHA" \
+            --title "ETLRelay ${PACKAGE_VERSION}" \
+            --generate-notes

etlrelay-0.1.0/.gitignore

@@ -0,0 +1,55 @@
+# Python bytecode and caches
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Python packaging / build artifacts
+build/
+dist/
+*.egg-info/
+.eggs/
+
+# Test and coverage artifacts
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+coverage.xml
+
+# Type checking / linting caches
+.pyright/
+.ruff_cache/
+
+# uv
+.uv/
+
+# IDEs and editors
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# macOS
+.DS_Store
+
+# Nix
+.direnv/
+result
+result-*
+
+# Local environment / secrets
+.env
+.env.*
+!.env.example
+
+# Local ETLRelay test data
+data/
+
+site/

etlrelay-0.1.0/CHANGELOG.md

@@ -0,0 +1,25 @@
+# Changelog
+
+Notable changes to ETLRelay are documented here.
+
+This project follows the principles of [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/).
+
+## [Unreleased]
+
+Changes intended for the first public release are listed below. When preparing the release, move this content to a `0.1.0` section and add the actual release date.
+
+### Added
+
+- Batch-oriented pipeline contracts for sources, processors, and sinks.
+- Synchronous pipeline execution with processors applied in sequence.
+- An Arrow-backed `Batch` representation.
+- Local filesystem access through a stream-based `FileSystem` protocol and `LocalFileSystem` implementation.
+- CSV reading and writing through separate reader and writer components.
+- `FileSource` and `FileSink` composition of storage and format components.
+- Path containment validation for local filesystem paths.
+- Unit, integration, and functional tests for the initial local CSV pipeline.
+
+### Notes
+
+- This is an experimental pre-1.0 release. Public APIs may change between minor releases.
+- The initial implementation focuses on local filesystem and CSV workflows. SQL, declarative YAML, CLI, Parquet, S3/HDFS, and generic API integrations are not part of this release unless separately implemented and tested.

etlrelay-0.1.0/DOCS_SETUP.md

@@ -0,0 +1,62 @@
+# MkDocs setup instructions for ETLRelay
+
+The original ETLRelay technology plan selected MkDocs for documentation, `uv` for Python dependency management, GitHub Actions for CI, and PyPI for distribution. MkDocs was selected in the plan but is not yet configured in the current documentation set. This package adds a minimal setup using Material for MkDocs for the theme and `mkdocstrings[python]` for API reference generation from source docstrings.
+
+## 1. Add documentation dependencies
+
+From the repository root, run:
+
+```bash
+uv add --group docs "mkdocs>=1.6,<2" "mkdocs-material>=9,<10" "mkdocstrings[python]>=0.18,<1"
+```
+
+This should add a `docs` dependency group to `pyproject.toml` and update `uv.lock`. Keep these packages out of runtime dependencies.
+
+Install the group:
+
+```bash
+uv sync --locked --group docs
+```
+
+## 2. Add the supplied files
+
+Copy `mkdocs.yml` and the `docs/` directory from this package into the repository root. Add `CHANGELOG.md`, `NOTICE`, and `SECURITY.md` at the repository root if they do not already exist. Merge rather than overwrite any existing project configuration.
+
+## 3. Ignore generated output
+
+Add this line to the repository `.gitignore` if it is not already present:
+
+```gitignore
+site/
+```
+
+## 4. Build and preview
+
+```bash
+uv run mkdocs serve
+uv run mkdocs build --strict
+```
+
+The API page references the expected `src/etlrelay` module layout. Review those import paths against the actual repository and fix any differences before considering the docs build complete.
+
+## 5. Integrate with existing CI
+
+Add this command to the existing GitHub Actions quality-check workflow after installing the `docs` dependency group:
+
+```bash
+uv run mkdocs build --strict
+```
+
+Do not create a duplicate CI workflow if an existing one already performs quality checks. A green strict build should be required for documentation changes.
+
+## 6. Finish release documents
+
+- `CHANGELOG.md`: move the initial-release entries under `[Unreleased]` to a `[0.1.0] - YYYY-MM-DD` heading with the actual release date when preparing the release tag.
+- `NOTICE`: confirm the copyright holder and year, and ensure the notice is consistent with the existing `LICENSE` file.
+- `SECURITY.md`: enable GitHub private vulnerability reporting before public release, or replace the reporting instructions with a verified private contact channel.
+- Getting Started: run `examples/copy_csv.py` against the built and installed package. Correct the example if actual signatures differ.
+- API reference: verify all paths listed in `docs/api-reference.md` exist in `src/etlrelay`.
+
+## Hosting
+
+MkDocs builds a static site in `site/`, which can be hosted on GitHub Pages or another static host. Hosting/deployment is intentionally not configured by this setup package because the repository URL, default branch, and GitHub Pages settings need to be confirmed in the actual repository. Add deployment after the strict local build passes.