kpops 4.1.1__tar.gz → 9.2.0.dev0__tar.gz

kpops-9.2.0.dev0/.bumpversion.cfg

@@ -0,0 +1,20 @@
+[bumpversion]
+current_version = 9.2.0-dev
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>dev)(?P<timestamp>\d+)?)?
+serialize = 
+	{major}.{minor}.{patch}-{release}{timestamp}
+	{major}.{minor}.{patch}-{release}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+first_value = dev
+optional_value = release
+values = 
+	dev
+	release
+
+[bumpversion:file:pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+
+[bumpversion:file:kpops/const/__init__.py]

kpops-9.2.0.dev0/.github/actions/update-docs/action.yml

@@ -0,0 +1,53 @@
+name: "Update documentation in gh-pages"
+description: |
+  Compile markdown documents to html and deploy to docs branch. If a semver version is given, this action strips the patch.
+  It then pushes the <major>.<minor> as the latest alias to the documentation branch with mike.
+
+inputs:
+  username:
+    description: "GitHub username"
+    required: true
+  email:
+    description: "GitHub email"
+    required: true
+  token:
+    description: "GitHub Token (must be a PAT for repository dispatch)"
+    required: true
+  version:
+    description: "Version name to be deployed by mike"
+    required: true
+  release:
+    description: "Determines if the set version is a stable and latest version, otherwise it is a dev version. (Default false)"
+    default: "false"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+      with:
+        python-version: "3.11"
+        version: "0.5.14"
+
+    - name: Update gh-pages branch
+      shell: bash
+      run: |
+        git config --local user.name  ${{ inputs.username }}
+        git config --local user.email ${{ inputs.email }}
+        git config --local user.password ${{ inputs.token }}
+        git fetch origin gh-pages
+
+    - name: Deploy ${{ inputs.version }} version of the documentation with mike
+      shell: bash
+      if: ${{ inputs.release == 'false' }}
+      run: |
+        uv run --frozen --group=docs mike deploy ${{ inputs.version }} --push --config-file ./docs/mkdocs.yml
+
+    - name: Deploy ${{ inputs.version }} version (latest) of the documentation with mike
+      shell: bash
+      if: ${{ inputs.release == 'true' }}
+      run: |
+        sem_version=${{ inputs.version }}
+        major_minor_version=${sem_version%.*}
+        uv run --frozen --group=docs mike deploy "$major_minor_version" latest --update-aliases --push --config-file ./docs/mkdocs.yml

kpops-9.2.0.dev0/.github/pyright-matcher.json

@@ -0,0 +1,18 @@
+{
+    "problemMatcher": [
+        {
+            "owner": "pyright",
+            "pattern": [
+                {
+                    "regexp": "^\\s\\s(\\S*\\.py):(\\d+):(\\d+)\\s-\\s(error|warning|information):\\s([\\s\\S]+?)\\s\\(([\\s\\S]+?)(?=(?:\\)|\\d+ errors,))",
+                    "file": 1,
+                    "line": 2,
+                    "column": 3,
+                    "severity": 4,
+                    "message": 5,
+                    "code": 6
+                }
+            ]
+        }
+    ]
+}

kpops-9.2.0.dev0/.github/workflows/_lint-gh-ci.yaml

@@ -0,0 +1,19 @@
+name: Lint GitHub CI
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/**
+      - actions/**
+
+jobs:
+  actionlint:
+    name: actionlint
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Lint actions and workflows
+        uses: bakdata/ci-templates/actions/action-lint@v1.14.0
+        with:
+          action-lint-version: "v1.7.6"

kpops-9.2.0.dev0/.github/workflows/add-issue-ci.yaml

@@ -0,0 +1,18 @@
+name: Add opened issues to specific project board
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-issue-to-project:
+    name: Add issue to project board
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Add the issue to the project
+        uses: actions/add-to-project@v0.4.0
+        with:
+          project-url: https://github.com/orgs/bakdata/projects/5
+          github-token: ${{ secrets.GH_TOKEN }}
+        id: add-project

kpops-9.2.0.dev0/.github/workflows/ci.yaml

@@ -0,0 +1,231 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, ready_for_review, synchronize]
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  UV_VERSION: "0.5.14"
+
+jobs:
+  lint:
+    name: Lint
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-24.04
+          - windows-2022
+        python-version: ["3.11", "3.12"]
+        is-draft-pr: 
+          - ${{ github.event.pull_request.draft }}
+        include:
+          - python-version: "3.11"
+            os: ubuntu-24.04
+            format-for-github: true
+        exclude:
+          # skip Windows jobs on draft PRs because they are slow
+          - os: windows-2022
+            is-draft-pr: true
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          version: ${{ env.UV_VERSION }}
+
+      - name: Check uv.lock
+        run: uv lock --check
+
+      - name: Lint (ruff)
+        run: |
+            uv run --frozen ruff check . --config pyproject.toml --no-fix ${{ matrix.format-for-github && '--output-format=github' }}
+
+      - name: Typing (pyright)
+        run: |
+          if [[ "${{ matrix.format-for-github }}" == "true" ]]
+          then
+            echo "::add-matcher::.github/pyright-matcher.json"
+          fi;
+          uv run --frozen pyright
+
+
+  format:
+    name: Format
+    runs-on: [ubuntu-24.04]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+          version: ${{ env.UV_VERSION }}
+
+      - name: ruff
+        run: uv run --frozen ruff format --check
+
+      - name: dprint
+        uses: dprint/check@v2.2
+
+  test:
+    name: Test
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-24.04
+          - windows-2022
+        python-version: ["3.11", "3.12"]
+        is-draft-pr: 
+          - ${{ github.event.pull_request.draft }}
+        exclude:
+          # skip Windows jobs on draft PRs because they are slow
+          - os: windows-2022
+            is-draft-pr: true
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          version: ${{ env.UV_VERSION }}
+
+      - name: Test
+        run: uv run --frozen pytest tests
+
+  docs:
+    name: Docs
+    runs-on: [ubuntu-24.04]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+          version: ${{ env.UV_VERSION }}
+
+      - name: Generate schema (kpops schema)
+        run: uv run --frozen pre-commit run gen-schema --all-files --show-diff-on-failure
+
+      - name: Generate CLI Usage docs (typer-cli)
+        run: uv run --frozen pre-commit run gen-docs-cli --all-files --show-diff-on-failure
+
+      - name: Generate Environment variable docs
+        run: uv run --frozen pre-commit run gen-docs-env-vars --all-files --show-diff-on-failure
+
+      - name: Generate pipeline definitions
+        run: uv run --frozen pre-commit run gen-docs-components --all-files --show-diff-on-failure
+
+      - name: Test docs build (mkdocs)
+        run: uv run --frozen --group=docs mkdocs build -f docs/mkdocs.yml
+
+  # TODO: extract into ci-templates
+  publish-snapshot-version:
+    name: Publish snapshot to TestPyPI
+    needs: [lint, format, test, docs]
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+          version: ${{ env.UV_VERSION }}
+
+      - name: Generate snapshot time
+        id: snapshot-time
+        run: |
+          snapshot_time=$(date +'%Y%m%d%H%M%S')
+          echo "snapshot-time=${snapshot_time}" >> "$GITHUB_OUTPUT"
+
+      - name: Bump version patch
+        id: bump-version
+        uses: bakdata/ci-templates/actions/bump-version@1.50.4
+        with:
+          release-type: patch
+
+      - name: Restore clean Git working directory
+        # clean Git working directory is needed for the next bumpversion call
+        # we only need the version result from the first `patch` bump which is stored as a step output
+        run: git restore . 
+
+      - name: Bump version snapshot time
+        uses: bakdata/ci-templates/actions/bump-version@1.50.4
+        with:
+          release-type: patch
+          new-version: ${{ steps.bump-version.outputs.release-version }}${{ steps.snapshot-time.outputs.snapshot-time }} # e.g. 9.1.1-dev20250108102827
+
+      - name: Build
+        run: uv build
+
+      - name: Publish TestPyPI
+        run: uv publish
+        env:
+          UV_PUBLISH_URL: https://test.pypi.org/legacy/
+          UV_PUBLISH_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }}
+
+  publish-docs-from-main:
+    name: Publish docs (main)
+    runs-on: ubuntu-24.04
+    if: ${{ github.ref == 'refs/heads/main' }}
+    needs: [lint, format, test, docs]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Publish docs from main branch
+        uses: ./.github/actions/update-docs
+        with:
+          username: ${{ secrets.GH_USERNAME }}
+          email: ${{ secrets.GH_EMAIL }}
+          token: ${{ secrets.GH_TOKEN }}
+          version: main
+
+  publish-dev-docs-from-pr:
+    name: Publish docs (dev)
+    runs-on: ubuntu-24.04
+    if: ${{ github.event_name == 'pull_request' }}
+    needs: [lint, format, test, docs]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      # Checks to see if any files in the PR match one of the listed file types.
+      # This will return true if there's a file in docs folder that was added, deleted, or modified in the PR.
+      - name: Check if files in docs folder have changed
+        uses: dorny/paths-filter@v3
+        id: docs-changes
+        with:
+          filters: |
+            docs:
+              - added|deleted|modified: 'docs/**'
+
+      - name: Publish dev docs from PR
+        if: steps.docs-changes.outputs.docs == 'true'
+        uses: ./.github/actions/update-docs
+        with:
+          username: ${{ secrets.GH_USERNAME }}
+          email: ${{ secrets.GH_EMAIL }}
+          token: ${{ secrets.GH_TOKEN }}
+          version: dev

kpops-9.2.0.dev0/.github/workflows/publish.yaml

@@ -0,0 +1,26 @@
+name: Publish to PyPI
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+          version: "0.5.14"
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        run: uv publish
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}

kpops-9.2.0.dev0/.github/workflows/release.yaml

@@ -0,0 +1,27 @@
+name: Release to GitHub and push tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      release-type:
+        description: "SemVer scope of the release"
+        type: choice
+        required: true
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  create-github-release-push-tag:
+    uses: bakdata/ci-templates/.github/workflows/bump-version-release.yaml@1.51.0
+    name: Release
+    with:
+      release-type: ${{ inputs.release-type }}
+      changelog: true
+      changelog-file: "./docs/docs/user/changelog.md"
+    secrets:
+      github-username: "${{ secrets.GH_USERNAME }}"
+      github-email: "${{ secrets.GH_EMAIL }}"
+      github-token: "${{ secrets.GH_TOKEN }}"

kpops-9.2.0.dev0/.github/workflows/update-gh-pages.yaml

@@ -0,0 +1,26 @@
+name: Update GitHub pages
+
+on:
+  push:
+    tags:
+      # Push events to semver tags
+      - '[0-9]+.[0-9]+.[0-9]+'
+
+jobs:
+  update-docs:
+    runs-on: ubuntu-24.04
+    name: Update docs
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all tags; Required for doc versioning
+
+      - name: Update gh-pages
+        uses: ./.github/actions/update-docs
+        with:
+          username: ${{ secrets.GH_USERNAME }}
+          email: ${{ secrets.GH_EMAIL }}
+          token: ${{ secrets.GH_TOKEN }}
+          version: ${{ github.ref_name }}
+          release: true

kpops-9.2.0.dev0/.gitignore

@@ -0,0 +1,7 @@
+.idea
+**/__pycache__
+.DS_Store
+.envrc
+.vscode
+site/
+scratch*

kpops-9.2.0.dev0/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "examples"]
+	path = examples
+	url = https://github.com/bakdata/kpops-examples

kpops-9.2.0.dev0/.pre-commit-config.yaml

@@ -0,0 +1,83 @@
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-lint
+        name: ruff-lint
+        entry: ruff check
+        args:
+          [
+            --force-exclude,
+            --config,
+            pyproject.toml,
+            --fix,
+            --show-fixes,
+            --exit-non-zero-on-fix,
+          ]
+        language: system
+        types_or: [python]
+        require_serial: true # run once for all files
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format
+        args: [--force-exclude, --config, pyproject.toml]
+        language: system
+        types_or: [python]
+        require_serial: true # run once for all files
+      - id: pyright
+        name: pyright
+        entry: pyright
+        language: system
+        types: [python]
+        require_serial: true # run once for all files
+      - id: gen-schema
+        name: gen-schema
+        entry: python -m hooks.gen_schema
+        language: system
+        types: [python]
+        require_serial: true
+        pass_filenames: false
+        exclude: ^tests/.*snapshots/
+      - id: gen-docs-cli
+        name: gen-docs-cli
+        entry: python -m hooks.gen_docs.gen_docs_cli_usage
+        language: system
+        types: [python]
+        require_serial: true
+        exclude: ^tests/.*snapshots/
+        pass_filenames: false
+      - id: gen-docs-env-vars
+        name: gen-docs-env-vars
+        entry: python -m hooks.gen_docs.gen_docs_env_vars
+        language: system
+        types: [python]
+        require_serial: true
+        exclude: ^tests/.*snapshots/
+        pass_filenames: false
+      - id: gen-docs-components
+        name: gen-docs-components
+        entry: python -m hooks.gen_docs.gen_docs_components
+        language: system
+        # .gitignore -- indicates whether the script is ran with `--all-files`
+        # hooks/gen_docs.py -- hook's behavior is possibly changed
+        # kpops/cli/main.py -- typer-cli has to generate the CLI Usage reference
+        # docs/.../.yaml -- examples changed or dependencies edited
+        # kpops/components/... -- KPOps components possibly changed
+        files: >
+          (?x)^(
+            .gitignore
+            | hooks/gen_docs\.py
+            | docs/docs/resources/pipeline-defaults/headers/.*\.yaml
+            | docs/docs/resources/pipeline-components/headers/.*\.yaml
+            | docs/docs/resources/pipeline-components/sections/.*\.yaml
+            | docs/docs/resources/pipeline-components/dependencies/.*\.yaml
+            | kpops/cli/main\.py
+            | kpops/components/.*\.py
+          )$
+        require_serial: true
+      - id: dprint
+        name: dprint
+        entry: dprint fmt
+        language: system
+        types: [markdown]
+        require_serial: true
+        pass_filenames: false

kpops-9.2.0.dev0/CHANGELOG.md

@@ -0,0 +1 @@
+docs/docs/user/changelog.md

kpops-9.2.0.dev0/CODEOWNERS

@@ -0,0 +1 @@
+* @disrupted

kpops-9.2.0.dev0/CONTRIBUTING.md

@@ -0,0 +1 @@
+docs/docs/developer/contributing.md

{kpops-4.1.1 → kpops-9.2.0.dev0}/PKG-INFO

@@ -1,39 +1,33 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: kpops
-Version: 4.1.1
+Version: 9.2.0.dev0
 Summary: KPOps is a tool to deploy Kafka pipelines to Kubernetes
-Home-page: https://github.com/bakdata/kpops
-License: MIT
-Keywords: kafka,kubernetes,stream-processing,pipelines
-Author: bakdata
-Author-email: opensource@bakdata.com
-Requires-Python: >=3.10,<3.13
+Author-email: bakdata <opensource@bakdata.com>
+License-File: LICENSE
+Keywords: kafka,kubernetes,pipelines,stream-processing
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Typing :: Typed
-Requires-Dist: PyYAML (>=6.0,<7.0)
-Requires-Dist: cachetools (>=5.2.0,<6.0.0)
-Requires-Dist: dictdiffer (>=0.9.0,<0.10.0)
-Requires-Dist: dtyper (>=2.1.0,<3.0.0)
-Requires-Dist: httpx (>=0.24.1,<1)
-Requires-Dist: networkx (>=3.1,<4.0)
-Requires-Dist: pydantic (>=2.5.3,<3.0.0)
-Requires-Dist: pydantic-settings (>=2.0.3,<3.0.0)
-Requires-Dist: pyhumps (>=3.7.3,<4.0.0)
-Requires-Dist: python-schema-registry-client (>=2.4.1,<3.0.0)
-Requires-Dist: rich (>=12.4.4,<13.0.0)
-Requires-Dist: typer[all] (>=0.6.1,<0.7.0)
-Project-URL: Documentation, https://bakdata.github.io/kpops/latest
-Project-URL: Repository, https://github.com/bakdata/kpops
+Requires-Python: >=3.11
+Requires-Dist: anyio>=4.3.0
+Requires-Dist: cachetools>=5.2.0
+Requires-Dist: croniter>=3.0.3
+Requires-Dist: dictdiffer>=0.9.0
+Requires-Dist: httpx>=0.24.1
+Requires-Dist: lightkube>=0.15.3
+Requires-Dist: networkx>=3.1
+Requires-Dist: pydantic-settings>=2.0.3
+Requires-Dist: pydantic>=2.5.3
+Requires-Dist: pyhumps>=3.7.3
+Requires-Dist: python-schema-registry-client>=2.4.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=12.4.4
+Requires-Dist: typer>=0.12.5
 Description-Content-Type: text/markdown
 
 # KPOps
@@ -63,7 +57,7 @@ the [documentation](https://bakdata.github.io/kpops/latest).
 KPOps comes as a [PyPI package](https://pypi.org/project/kpops/).
 You can install it with [pip](https://github.com/pypa/pip):
 
-```shell
+```sh
 pip install kpops
 ```
 
@@ -81,4 +75,3 @@ Just open an issue beforehand and let us know what you want to do and why.
 ## License
 
 KPOps is licensed under the [MIT License](https://github.com/bakdata/kpops/blob/main/LICENSE).
-

{kpops-4.1.1 → kpops-9.2.0.dev0}/README.md

@@ -25,7 +25,7 @@ the [documentation](https://bakdata.github.io/kpops/latest).
 KPOps comes as a [PyPI package](https://pypi.org/project/kpops/).
 You can install it with [pip](https://github.com/pypa/pip):
 
-```shell
+```sh
 pip install kpops
 ```