loom-kernel 1.9.2__tar.gz → 1.10.0__tar.gz

loom_kernel-1.10.0/.github/workflows/ci-main.yml

@@ -0,0 +1,403 @@
+name: ci-main
+
+on:
+  push:
+    branches: ["master"]
+  # The release helper merges the version bump through the API with
+  # GITHUB_TOKEN, and GitHub deliberately emits no `push` for it, so that commit
+  # would otherwise reach master without ever being validated. `workflow_dispatch`
+  # is the documented exception that does start a run from that token, so the
+  # release asks for this workflow by hand right after it merges.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    env:
+      SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
+      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+      SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+      SONAR_PROJECT_KEY: ${{ vars.SONAR_PROJECT_KEY }}
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.11"
+
+      # uv is pinned to the same version in every workflow to avoid drift, and
+      # installed from wheels only so no sdist setup script can run.
+      - name: Install uv
+        run: |
+          pip install --only-binary :all: "uv==0.10.2"
+
+      - name: Quality report (strict)
+        uses: the-reacher-data/loom-actions/actions/python/quality-report@bdb1b5bc76f37f3f33a6598224e864c17b8d756a # v1.0.5
+        with:
+          src-dir: src
+          test-dir: tests
+          coverage-threshold: "80"
+          fail-on-quality: "any"
+          fail-on-security: "high"
+          include-security: "true"
+
+      # --frozen forces the committed uv.lock resolution and pytest-cov is
+      # pinned to the version already locked in the dev group.
+      # "--no-build" is intentionally omitted: omegaconf (a runtime dependency)
+      # requires antlr4-python3-runtime 4.9.3, which ships no wheel on PyPI, so
+      # the environment cannot be created from binaries only.
+      # Fast lane: `-m "not integration"` deselects the suites that need a
+      # provisioned external service; they run in the `integration` job with
+      # those services actually up. Deselecting is deliberate over letting them
+      # skip — a skipped test reads as approved, a deselected one reads as
+      # "not this lane".
+      # Standalone artifact validation (FR-015): the published corpus must
+      # stay valid against the offline validator. The fake packages provide
+      # the corpus module:symbol references, as they would in an application.
+      # The glob is '*/agent.yaml', not '*.yaml': each corpus entry is a
+      # DIRECTORY holding its artifact next to its own skills, which is the
+      # layout an application uses. A top-level '*.yaml' matches nothing, and
+      # the validator exits 0 on an empty match — so the wrong glob is a step
+      # that passes without validating anything.
+      - name: Validate agent artifact corpus
+        shell: bash
+        env:
+          PYTHONPATH: tests/unit/ai/fixtures/fake_pkgs
+        run: |
+          set -euo pipefail
+          uv run --frozen python -m loom.ai.validate 'tests/unit/ai/fixtures/corpus_v1/*/agent.yaml'
+
+      - name: Run pytest with coverage and JUnit output
+        shell: bash
+        run: |
+          set -euo pipefail
+          uv run --frozen --with "pytest-cov==7.1.0" \
+            pytest -m "not integration" --cov --junitxml=junit.xml -o junit_family=legacy
+
+      - name: Upload coverage to Codecov
+        if: ${{ hashFiles('coverage.xml') != '' }}
+        uses: codecov/codecov-action@671740ac38dd9b0130fbe1cec585b89eea48d3de # v5
+        with:
+          files: ./coverage.xml
+          fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() && hashFiles('junit.xml') != '' }}
+        uses: codecov/test-results-action@0fa95f0e1eeaafde2c782583b36b28ad0d8c77d3 # v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Require SonarQube analysis configuration
+        shell: bash
+        run: |
+          set -euo pipefail
+          missing=""
+          [ -n "${SONAR_TOKEN}" ] || missing="${missing} SONAR_TOKEN(secret)"
+          [ -n "${SONAR_HOST_URL}" ] || missing="${missing} SONAR_HOST_URL(secret)"
+          [ -n "${SONAR_PROJECT_KEY}" ] || missing="${missing} SONAR_PROJECT_KEY(variable)"
+          if [ -n "${missing}" ]; then
+            echo "SonarQube scan cannot run, missing:${missing}"
+            echo "Without it the scan is skipped and the only analysis left is"
+            echo "SonarCloud Automatic Analysis, which ignores sonar-project.properties:"
+            echo "exclusions and issue suppressions silently stop applying."
+            exit 1
+          fi
+
+      - name: SonarQube scan
+        if: ${{ env.SONAR_TOKEN != '' && env.SONAR_HOST_URL != '' && env.SONAR_PROJECT_KEY != '' }}
+        uses: SonarSource/sonarqube-scan-action@2f77a1ec69fb1d595b06f35ab27e97605bdef703 # v5
+        with:
+          args: >
+            -Dsonar.projectKey=${{ env.SONAR_PROJECT_KEY }}
+        env:
+          SONAR_TOKEN: ${{ env.SONAR_TOKEN }}
+          SONAR_HOST_URL: ${{ env.SONAR_HOST_URL }}
+
+      - name: Snyk dependency monitor
+        if: ${{ env.SNYK_TOKEN != '' }}
+        uses: snyk/actions/setup@9adf32b1121593767fc3c057af55b55db032dc04 # master
+        env:
+          SNYK_TOKEN: ${{ env.SNYK_TOKEN }}
+
+      # Non-blocking, exactly as the same scan is on a pull request: a scanner
+      # that is unreachable or refusing an expired token says nothing about this
+      # code, and master turning red for it hides the failures that do mean
+      # something. The outcome is reported loudly in the step below instead.
+      - name: Snyk monitor by dependency scope (core/dev/extras)
+        id: snyk
+        if: ${{ env.SNYK_TOKEN != '' }}
+        continue-on-error: true
+        shell: bash
+        env:
+          SNYK_TOKEN: ${{ env.SNYK_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          MANIFEST_DIR="$RUNNER_TEMP/snyk-manifests"
+          mkdir -p "$MANIFEST_DIR"
+
+          # Two exports of the same locked resolution: Snyk reads the plain
+          # manifest, pip installs the hashed twin so every downloaded artifact
+          # is verified against uv.lock.
+          uv export --frozen --no-default-groups --no-dev --no-hashes --no-emit-project \
+            -o "$MANIFEST_DIR/requirements-core.txt"
+          uv export --frozen --no-default-groups --no-dev --no-emit-project \
+            -o "$MANIFEST_DIR/hashed-core.txt"
+          uv export --frozen --only-group dev --no-hashes --no-emit-project \
+            -o "$MANIFEST_DIR/requirements-dev.txt"
+          uv export --frozen --only-group dev --no-emit-project \
+            -o "$MANIFEST_DIR/hashed-dev.txt"
+
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
+            uv export --frozen --no-default-groups --no-dev --extra "$extra" \
+              --no-hashes --no-emit-project \
+              -o "$MANIFEST_DIR/requirements-extra-${extra}.txt"
+            uv export --frozen --no-default-groups --no-dev --extra "$extra" \
+              --no-emit-project \
+              -o "$MANIFEST_DIR/hashed-extra-${extra}.txt"
+          done
+
+          python -m pip install --require-hashes -r "$MANIFEST_DIR/hashed-core.txt"
+          snyk monitor --skip-unresolved \
+            --file="$MANIFEST_DIR/requirements-core.txt" --package-manager=pip \
+            --project-name="loom-py/core" --target-reference="${GITHUB_REF_NAME:-master}"
+          python -m pip install --require-hashes -r "$MANIFEST_DIR/hashed-dev.txt"
+          snyk monitor --skip-unresolved \
+            --file="$MANIFEST_DIR/requirements-dev.txt" --package-manager=pip \
+            --project-name="loom-py/dev" --target-reference="${GITHUB_REF_NAME:-master}"
+
+          for extra in rest sqlalchemy config cache pyspark prometheus celery streaming; do
+            python -m pip install --require-hashes -r "$MANIFEST_DIR/hashed-extra-${extra}.txt"
+            snyk monitor --skip-unresolved \
+              --file="$MANIFEST_DIR/requirements-extra-${extra}.txt" \
+              --package-manager=pip \
+              --project-name="loom-py/extra-${extra}" \
+              --target-reference="${GITHUB_REF_NAME:-master}"
+          done
+
+      - name: Report the Snyk outcome
+        if: ${{ always() && env.SNYK_TOKEN != '' }}
+        env:
+          OUTCOME: ${{ steps.snyk.outcome }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          {
+            echo "## Snyk dependency scan: \`${OUTCOME}\`"
+          } >> "$GITHUB_STEP_SUMMARY"
+          if [ "${OUTCOME}" != "success" ]; then
+            {
+              echo
+              echo "The scan did not complete, so **no dependency was cleared and none was"
+              echo "reported vulnerable**. This does NOT block the build: a scanner that is"
+              echo "unreachable, rate-limited or refusing an expired \`SNYK_TOKEN\` says"
+              echo "nothing about this code. Read the step log: an \`Authentication error"
+              echo "(SNYK-0005)\` with \`401 Unauthorized\` means the token needs renewing"
+              echo "in Snyk and updating in the repository secrets."
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::warning title=Snyk scan did not complete::No dependency was scanned. This does NOT block the build. A 401 with SNYK-0005 means the SNYK_TOKEN is expired or unprovisioned; renew it in Snyk and update the repository secret."
+          fi
+
+  # ---------------------------------------------------------------------------
+  # Churn canary (plan.md risk table): run the agent suites against the LATEST
+  # published pydantic-ai instead of the locked one.
+  #
+  # Why this job exists, concretely. During this feature three separate breaks
+  # were found by hand, each one the package disagreeing with its own docs:
+  #   * `Agent.to_a2a` had been removed;
+  #   * `FunctionToolResultEvent.result` did not exist;
+  #   * `Skills.include` accepted exact names only, not the patterns documented.
+  # Each cost a debugging session. The fourth should cost a CI annotation, and
+  # it should arrive on the day upstream ships it, not on the day we upgrade.
+  #
+  # Non-blocking by construction: `continue-on-error` on the job so a red canary
+  # never gates a PR, and a loud `::warning::` annotation plus a job summary so
+  # it is never a silent grey tick either. A canary nobody sees is not a canary.
+  #
+  # No `--frozen` and no `uv.lock` here — resolving fresh is the entire point.
+  pydantic-ai-canary:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: |
+          pip install --only-binary :all: "uv==0.10.2"
+
+      - name: Configure uv environment path
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/uv-ci-canary" >> "$GITHUB_ENV"
+
+      # Sync the locked environment first so only the pydantic-ai family moves:
+      # a canary that also floats msgspec or fastapi would report churn without
+      # saying whose churn it is.
+      - name: Sync the locked environment
+        shell: bash
+        run: |
+          set -euo pipefail
+          uv sync --frozen
+
+      # Two steps rather than one `--upgrade`, on purpose. The repository's
+      # supply-chain contract requires every CI install to name an exact
+      # version and to refuse sdists, so nothing can execute a setup script on
+      # a runner. A canary that floated its version would break that rule; a
+      # canary that RESOLVES the latest version, records it, and then installs
+      # that exact pin from wheels keeps it — and gains reproducibility, since
+      # the log now says precisely which release was tested.
+      - name: Resolve the latest published versions
+        id: resolve
+        shell: bash
+        run: |
+          set -euo pipefail
+          for package in pydantic-ai-slim pydantic-ai-harness fasta2a a2a-sdk; do
+            version="$(python -c 'import json,sys,urllib.request; print(json.load(urllib.request.urlopen("https://pypi.org/pypi/" + sys.argv[1] + "/json", timeout=30))["info"]["version"])' "$package")"
+            echo "Latest ${package}: ${version}"
+            echo "${package//-/_}=${version}" >> "$GITHUB_ENV"
+          done
+
+      - name: Install the latest release, pinned and wheels-only
+        id: upgrade
+        shell: bash
+        run: |
+          set -euo pipefail
+          # --python is required: uv pip honours VIRTUAL_ENV, not
+          # UV_PROJECT_ENVIRONMENT, so without it the install cannot find the
+          # environment 'uv sync' just created and fails asking for a venv.
+          uv pip install --python "$UV_PROJECT_ENVIRONMENT" --only-binary :all: \
+            "pydantic-ai-slim[anthropic,bedrock,openai]==${pydantic_ai_slim}" \
+            "pydantic-ai-harness[skills]==${pydantic_ai_harness}" \
+            "fasta2a==${fasta2a}" \
+            "a2a-sdk==${a2a_sdk}"
+          {
+            echo "resolved<<EOF"
+            echo "pydantic-ai-slim==${pydantic_ai_slim}"
+            echo "pydantic-ai-harness==${pydantic_ai_harness}"
+            echo "fasta2a==${fasta2a}"
+            echo "a2a-sdk==${a2a_sdk}"
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Run the agent suites against the latest release
+        id: canary
+        continue-on-error: true
+        shell: bash
+        run: |
+          set -euo pipefail
+          uv run --no-sync pytest tests/unit/ai tests/contract/ai tests/integration/ai -q
+
+      # Always reports, pass or fail: the version tested is as useful as the
+      # verdict, because it is what pins the upgrade when the verdict is red.
+      - name: Report the canary verdict
+        if: ${{ always() }}
+        shell: bash
+        env:
+          RESOLVED: ${{ steps.upgrade.outputs.resolved }}
+          OUTCOME: ${{ steps.canary.outcome }}
+        run: |
+          set -euo pipefail
+          {
+            echo "## pydantic-ai churn canary: \`${OUTCOME}\`"
+            echo
+            echo "Versions under test:"
+            echo '```'
+            echo "${RESOLVED}"
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+          if [ "${OUTCOME}" != "success" ]; then
+            echo "::warning title=pydantic-ai churn canary failed::The agent suites fail against the latest pydantic-ai. This does NOT block the PR. It means the next upgrade of the pinned version needs work first — read the job log, then either adapt the adapter or hold the pin."
+          fi
+
+  # ---------------------------------------------------------------------------
+  # Integration lane: everything marked `integration` needs a real external
+  # service, so it runs here with the services actually provisioned — never as a
+  # silent skip inside the fast lane. The same docker-compose.local.yaml that
+  # developers use locally is the single source of truth for how these services
+  # are configured.
+  #
+  # The Kafka suite runs twice: Redpanda is the fast loop, Apache Kafka in KRaft
+  # mode is the fidelity check. The partitioned source uses the consumer group
+  # purely as an offset store (assign, never subscribe), and that is exactly the
+  # area where broker implementations legitimately diverge — a green Redpanda run
+  # is not evidence that Apache Kafka agrees.
+  # ---------------------------------------------------------------------------
+  integration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: |
+          pip install --only-binary :all: "uv==0.10.2"
+
+      - name: Configure uv environment path
+        shell: bash
+        run: |
+          set -euo pipefail
+          echo "UV_PROJECT_ENVIRONMENT=$RUNNER_TEMP/uv-ci-integration" >> "$GITHUB_ENV"
+
+      - name: Start integration services
+        shell: bash
+        run: |
+          set -euo pipefail
+          docker compose -f docker-compose.local.yaml up -d redpanda kafka clickhouse
+
+      - name: Wait for services to report healthy
+        shell: bash
+        run: |
+          set -euo pipefail
+          for service in redpanda kafka clickhouse; do
+            container="$(docker compose -f docker-compose.local.yaml ps -q "$service")"
+            for _ in $(seq 1 60); do
+              status="$(docker inspect -f '{{.State.Health.Status}}' "$container")"
+              [ "$status" = "healthy" ] && break
+              sleep 2
+            done
+            if [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" != "healthy" ]; then
+              echo "::error::$service never became healthy"
+              docker compose -f docker-compose.local.yaml logs "$service"
+              exit 1
+            fi
+          done
+
+      - name: Integration tests (Redpanda)
+        shell: bash
+        env:
+          LOOM_KAFKA_IT_BOOTSTRAP: localhost:19092
+        run: |
+          set -euo pipefail
+          uv run --frozen pytest -m integration -ra
+
+      - name: Integration tests (Apache Kafka fidelity check)
+        shell: bash
+        env:
+          LOOM_KAFKA_IT_BOOTSTRAP: localhost:19093
+        run: |
+          set -euo pipefail
+          uv run --frozen pytest tests/integration/streaming/kafka -ra
+
+      - name: Dump service logs on failure
+        if: ${{ failure() }}
+        shell: bash
+        run: docker compose -f docker-compose.local.yaml logs --no-color --tail=200
+
+      - name: Stop integration services
+        if: ${{ always() }}
+        shell: bash
+        run: docker compose -f docker-compose.local.yaml down -v