strands-agents-evals 0.1.0__tar.gz

strands_agents_evals-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,104 @@
+name: Bug Report
+description: Report a bug in the Strands Agents Evals
+title: "[BUG] "
+labels: ["bug", "triage"]
+assignees: []
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report for Strands Agents Evals!
+  - type: checkboxes
+    id: "checks"
+    attributes:
+      label: "Checks"
+      options:
+        - label: "I have updated to the lastest minor and patch version of Strands and evals"
+          required: true
+        - label: "I have checked the documentation and this is not expected behavior"
+          required: true
+        - label: "I have searched [./issues](./issues?q=) and there are no duplicates of my issue"
+          required: true
+  - type: input
+    id: strands-version
+    attributes:
+      label: Strands Version
+      description: Which version of Strands are you using?
+      placeholder: e.g., 0.5.2
+    validations:
+      required: true
+  - type: input
+    id: strands-evals-version
+    attributes:
+      label: Strands Evals Version
+      description: Which version of Strands Evals are you using?
+      placeholder: e.g., 0.5.2
+    validations:
+      required: true
+  - type: input
+    id: python-version
+    attributes:
+      label: Python Version
+      description: Which version of Python are you using?
+      placeholder: e.g., 3.10.5
+    validations:
+      required: true
+  - type: input
+    id: os
+    attributes:
+      label: Operating System
+      description: Which operating system are you using?
+      placeholder: e.g., macOS 12.6
+    validations:
+      required: true
+  - type: dropdown
+    id: installation-method
+    attributes:
+      label: Installation Method
+      description: How did you install Strands?
+      options:
+        - pip
+        - git clone
+        - binary
+        - other
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Detailed steps to reproduce the behavior
+      placeholder: |
+        1. Code Snippet (Minimal reproducible example)
+        2. Install Strands using...
+        3. Run the command...
+        4. See error...
+    validations:
+      required: true
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: A clear description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual Behavior
+      description: What actually happened
+    validations:
+      required: true
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Any other relevant information, logs, screenshots, etc.
+  - type: textarea
+    id: possible-solution
+    attributes:
+      label: Possible Solution
+      description: Optional - If you have suggestions on how to fix the bug
+  - type: input
+    id: related-issues
+    attributes:
+      label: Related Issues
+      description: Optional - Link to related issues if applicable

strands_agents_evals-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Strands Evals SDK Documentation
+    url: https://github.com/strands-agents/docs
+    about: Visit our documentation for help

strands_agents_evals-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,41 @@
+name: Feature Request
+description: Suggest a new feature or enhancement for Strands Evals SDK
+title: "[FEATURE] "
+labels: ["enhancement", "triage"]
+assignees: []
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for suggesting a new feature for Strands Evals SDK!
+  - type: textarea
+    id: problem-statement
+    attributes:
+      label: Problem Statement
+      description: Describe the problem you're trying to solve. What is currently difficult or impossible to do?
+      placeholder: I would like Strands Evals to...
+    validations:
+      required: true
+  - type: textarea
+    id: proposed-solution
+    attributes:
+      label: Proposed Solution
+      description: Optional - Describe your proposed solution in detail. How would this feature work?
+  - type: textarea
+    id: use-case
+    attributes:
+      label: Use Case
+      description: Provide specific use cases for the feature. How would people use it?
+      placeholder: This would help with...
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives-solutions
+    attributes:
+      label: Alternatives Solutions
+      description: Optional - Have you considered alternative approaches? What are their pros and cons?
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      description: Include any other context, screenshots, code examples, or references that might help understand the feature request.

strands_agents_evals-0.1.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,38 @@
+## Description
+<!-- Provide a detailed description of the changes in this PR -->
+
+## Related Issues
+
+<!-- Link to related issues using #issue-number format -->
+
+## Documentation PR
+
+<!-- Link to related associated PR in the agent-docs repo -->
+
+## Type of Change
+
+<!-- Choose one of the following types of changes, delete the rest -->
+
+Bug fix
+New feature
+Breaking change
+Documentation update
+Other (please describe):
+
+## Testing
+
+How have you tested the change?  Verify that the changes do not break functionality or introduce warnings in consuming repositories: agents-docs, agents-tools, agents-cli
+
+- [ ] I ran `hatch run prepare`
+
+## Checklist
+- [ ] I have read the CONTRIBUTING document
+- [ ] I have added any necessary tests that prove my fix is effective or my feature works
+- [ ] I have updated the documentation accordingly
+- [ ] I have added an appropriate example to the documentation to outline the feature, or no new docs are needed
+- [ ] My changes generate no new warnings
+- [ ] Any dependent changes have been merged and published
+
+----
+
+By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

strands_agents_evals-0.1.0/.github/dependabot.yml

@@ -0,0 +1,20 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 100
+    commit-message:
+      prefix: ci
+    groups:
+      dev-dependencies:
+        patterns:
+          - "pytest"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    open-pull-requests-limit: 100
+    commit-message:
+      prefix: ci

strands_agents_evals-0.1.0/.github/workflows/integration-test.yml

@@ -0,0 +1,73 @@
+name: Secure Integration test
+
+on:
+  pull_request_target:
+    branches: main
+
+jobs:
+  authorization-check:
+    permissions: read-all
+    runs-on: ubuntu-latest
+    outputs:
+      approval-env: ${{ steps.collab-check.outputs.result }}
+    steps:
+      - name: Collaborator Check
+        uses: actions/github-script@v8
+        id: collab-check
+        with:
+          result-encoding: string
+          script: |
+            try {
+              const permissionResponse = await github.rest.repos.getCollaboratorPermissionLevel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                username: context.payload.pull_request.user.login,
+              });
+              const permission = permissionResponse.data.permission;
+              const hasWriteAccess = ['write', 'admin'].includes(permission);
+              if (!hasWriteAccess) {
+                console.log(`User ${context.payload.pull_request.user.login} does not have write access to the repository (permission: ${permission})`);
+                return "manual-approval"
+              } else {
+                console.log(`Verifed ${context.payload.pull_request.user.login} has write access. Auto Approving PR Checks.`)
+                return "auto-approve"
+              }
+            } catch (error) {
+              console.log(`${context.payload.pull_request.user.login} does not have write access. Requiring Manual Approval to run PR Checks.`)
+              return "manual-approval"
+            }
+  check-access-and-checkout:
+    runs-on: ubuntu-latest
+    needs: authorization-check
+    environment: ${{ needs.authorization-check.outputs.approval-env }}
+    permissions:
+      id-token: write
+      pull-requests: read
+      contents: read
+    steps:
+      - name: Configure Credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+         role-to-assume: ${{ secrets.STRANDS_INTEG_TEST_ROLE }}
+         aws-region: us-east-1
+         mask-aws-account-id: true
+      - name: Checkout head commit
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }} # Pull the commit from the forked repo
+          persist-credentials: false  # Don't persist credentials for subsequent actions
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir hatch
+      - name: Run integration tests
+        env:
+          AWS_REGION: us-east-1
+          AWS_REGION_NAME: us-east-1
+          STRANDS_TEST_API_KEYS_SECRET_NAME: ${{ secrets.STRANDS_TEST_API_KEYS_SECRET_NAME }}
+        id: tests
+        run: |
+          hatch test tests_integ

strands_agents_evals-0.1.0/.github/workflows/pr-and-push.yml

@@ -0,0 +1,19 @@
+name: Pull Request and Push Action
+
+on:
+  pull_request:  # Safer than pull_request_target for untrusted code
+    branches: [ main ]
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [ main ]  # Also run on direct pushes to main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  call-test-lint:
+    uses: ./.github/workflows/test-lint.yml
+    permissions:
+      contents: read
+    with:
+      ref: ${{ github.event.pull_request.head.sha }}

strands_agents_evals-0.1.0/.github/workflows/pypi-publish-on-release.yml

@@ -0,0 +1,82 @@
+name: Publish Python Package
+
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  call-test-lint:
+    uses: ./.github/workflows/test-lint.yml
+    permissions:
+      contents: read
+    with:
+      ref: ${{ github.event.release.target_commitish }}
+
+  build:
+    name: Build distribution 📦
+    permissions:
+      contents: read
+    needs:
+      - call-test-lint
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.10'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install hatch twine
+
+    - name: Validate version
+      run: |
+        version=$(hatch version)
+        if [[ $version =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "Valid version format"
+            exit 0
+        else
+            echo "Invalid version format"
+            exit 1
+        fi
+
+    - name: Build
+      run: |
+        hatch build
+
+    - name: Store the distribution packages
+      uses: actions/upload-artifact@v5
+      with:
+        name: python-package-distributions
+        path: dist/
+
+  deploy:
+    name: Upload release to PyPI
+    needs:
+    - build
+    runs-on: ubuntu-latest
+
+    # environment is used by PyPI Trusted Publisher and is strongly encouraged
+    # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+    environment:
+      name: pypi
+      url: https://pypi.org/p/strands-agents-evals
+    permissions:
+      # IMPORTANT: this permission is mandatory for Trusted Publishing
+      id-token: write
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1

strands_agents_evals-0.1.0/.github/workflows/test-lint.yml

@@ -0,0 +1,94 @@
+name: Test and Lint
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        required: true
+        type: string
+
+jobs:
+  unit-test:
+    name: Unit Tests - Python ${{ matrix.python-version }} - ${{ matrix.os-name }}
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+       include:
+        # Linux
+        - os: ubuntu-latest
+          os-name: 'linux'
+          python-version: "3.10"
+        - os: ubuntu-latest
+          os-name: 'linux'
+          python-version: "3.11"
+        - os: ubuntu-latest
+          os-name: 'linux'
+          python-version: "3.12"
+        - os: ubuntu-latest
+          os-name: 'linux'
+          python-version: "3.13"
+        # Windows
+        - os: windows-latest
+          os-name: 'windows'
+          python-version: "3.10"
+        - os: windows-latest
+          os-name: 'windows'
+          python-version: "3.11"
+        - os: windows-latest
+          os-name: 'windows'
+          python-version: "3.12"
+        - os: windows-latest
+          os-name: 'windows'
+          python-version: "3.13"
+        # MacOS - latest only; not enough runners for macOS
+        - os: macos-latest
+          os-name: 'macOS'
+          python-version: "3.13"
+      fail-fast: true
+    runs-on: ${{ matrix.os }}
+    env:
+      LOG_LEVEL: DEBUG
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref }}  # Explicitly define which commit to check out
+          persist-credentials: false  # Don't persist credentials for subsequent actions
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir hatch
+      - name: Run Unit tests
+        id: tests
+        run: hatch test tests --cover
+        continue-on-error: false
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ inputs.ref }}
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir hatch
+
+      - name: Run lint
+        id: lint
+        run: hatch run test-lint
+        continue-on-error: false

strands_agents_evals-0.1.0/.gitignore

@@ -0,0 +1,17 @@
+__pycache__*
+.pytest_cache
+slack_events
+build
+.coverage*
+.env
+.venv
+.mypy_cache
+.ruff_cache
+*.bak
+.vscode
+dist
+repl_state
+dataset_files
+report_files
+.venv
+*.DS_Store*

strands_agents_evals-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+  - repo: local
+    hooks:
+      - id: hatch-format
+        name: Format code
+        entry: hatch fmt --formatter --check
+        language: system
+        pass_filenames: false
+        types: [python]
+        stages: [pre-commit]
+      - id: hatch-lint
+        name: Lint code
+        entry: hatch fmt --linter --check
+        language: system
+        pass_filenames: false
+        types: [python]
+        stages: [pre-commit]
+      - id: hatch-test
+        name: Unit tests
+        entry: hatch test
+        language: system
+        pass_filenames: false
+        types: [python]
+        stages: [pre-commit]
+      - id: commitizen-check
+        name: Check commit message
+        entry: hatch run cz check --commit-msg-file
+        language: system
+        stages: [commit-msg]

strands_agents_evals-0.1.0/CODE_OF_CONDUCT.md

@@ -0,0 +1,4 @@
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
+opensource-codeofconduct@amazon.com with any additional questions or comments.