splunk-soar-sdk 3.2.3__tar.gz → 3.3.0__tar.gz

splunk_soar_sdk-3.3.0/.github/scripts/generate_test_summary.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Generate integration test summary from JUnit XML reports."""
+
+import sys
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class TestResult:
+    """Test result summary for a single version."""
+
+    version: str
+    passed: int
+    failed: int
+    skipped: int
+    errors: int
+    duration: float
+    failures: list[dict[str, str]]
+
+
+def parse_junit_xml(xml_path: Path) -> TestResult:
+    """Parse JUnit XML file and extract test results."""
+    tree = ET.parse(xml_path)  # noqa: S314
+    root = tree.getroot()
+
+    version = xml_path.stem.replace("junit-", "")
+    passed = 0
+    failed = 0
+    skipped = 0
+    errors = 0
+    duration = 0.0
+    failures = []
+
+    for testsuite in root.findall(".//testsuite"):
+        duration += float(testsuite.get("time", 0))
+        passed += (
+            int(testsuite.get("tests", 0))
+            - int(testsuite.get("failures", 0))
+            - int(testsuite.get("skipped", 0))
+            - int(testsuite.get("errors", 0))
+        )
+        failed += int(testsuite.get("failures", 0))
+        skipped += int(testsuite.get("skipped", 0))
+        errors += int(testsuite.get("errors", 0))
+
+    for testcase in root.findall(".//testcase"):
+        failure = testcase.find("failure")
+        error = testcase.find("error")
+
+        if failure is not None:
+            failures.append(
+                {
+                    "name": f"{testcase.get('classname')}.{testcase.get('name')}",
+                    "message": failure.get("message", "No message"),
+                    "type": failure.get("type", "AssertionError"),
+                    "output": (failure.text or "")[:500],
+                }
+            )
+        elif error is not None:
+            failures.append(
+                {
+                    "name": f"{testcase.get('classname')}.{testcase.get('name')}",
+                    "message": error.get("message", "No message"),
+                    "type": error.get("type", "Error"),
+                    "output": (error.text or "")[:500],
+                }
+            )
+
+    return TestResult(
+        version=version,
+        passed=passed,
+        failed=failed,
+        skipped=skipped,
+        errors=errors,
+        duration=duration,
+        failures=failures,
+    )
+
+
+def format_duration(seconds: float) -> str:
+    """Format duration in human-readable format."""
+    minutes = int(seconds // 60)
+    secs = int(seconds % 60)
+    return f"{minutes}m {secs}s"
+
+
+def generate_markdown_summary(
+    results: list[TestResult], run_url: str | None = None
+) -> str:
+    """Generate markdown summary from test results."""
+    total_passed = sum(r.passed for r in results)
+    total_failed = sum(r.failed for r in results)
+    total_errors = sum(r.errors for r in results)
+
+    md = "# Integration Test Results\n"
+
+    if total_failed + total_errors == 0:
+        md += f"## ✅ {total_passed} tests passed!\n"
+    else:
+        md += "## ⚠️ Some tests failed.\n"
+
+    md += "| Version | ✅ Passed | ❌ Failed | ⏭️ Skipped | Duration |\n"
+    md += "| ------- | --------- | --------- | ---------- | -------- |\n"
+
+    for result in sorted(results, key=lambda r: r.version):
+        md += f"| {result.version} | {result.passed} | {result.failed + result.errors} | {result.skipped} | {format_duration(result.duration)} |\n"
+
+    has_failures = any(r.failures for r in results)
+    if has_failures:
+        md += "## Failed Tests\n"
+        md += "| Version | Name | Message | Type |\n"
+        md += "| ------- | ---- | ------- | ---- |\n"
+        for result in results:
+            if result.failures:
+                for failure in result.failures:
+                    message = failure["message"]
+                    if len(message) > 200:
+                        message = message[:200].strip() + "…"
+                    md += f"| {result.version} | {failure['name']} | {message} | {failure['type']} |\n"
+
+    if run_url:
+        md += "---\n[View full logs and artifacts](" + run_url + ")\n"
+
+    return md
+
+
+def main() -> int:
+    """Main entry point."""
+    if len(sys.argv) < 2:
+        print(
+            "Usage: generate_test_summary.py <test-results-dir> [output-file] [run-url]"
+        )
+        return 1
+
+    test_results_dir = Path(sys.argv[1])
+    if not test_results_dir.exists():
+        print(f"Error: Directory {test_results_dir} does not exist")
+        return 1
+
+    junit_files = list(test_results_dir.glob("**/junit-*.xml"))
+    if not junit_files:
+        print(f"Error: No JUnit XML files found in {test_results_dir}")
+        return 1
+
+    results = []
+    for junit_file in junit_files:
+        try:
+            result = parse_junit_xml(junit_file)
+            results.append(result)
+        except Exception as e:
+            print(f"Error parsing {junit_file}: {e}")
+            continue
+
+    if not results:
+        print("Error: No test results could be parsed")
+        return 1
+
+    run_url = sys.argv[3] if len(sys.argv) > 3 else None
+    summary = generate_markdown_summary(results, run_url)
+    print(summary)
+
+    summary_file = Path(sys.argv[2]) if len(sys.argv) > 2 else Path("test-summary.md")
+    summary_file.write_text(summary)
+    print(f"\nSummary written to {summary_file}", file=sys.stderr)
+
+    has_failures = any(r.failed + r.errors > 0 for r in results)
+    return 1 if has_failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

{splunk_soar_sdk-3.2.3 → splunk_soar_sdk-3.3.0}/.github/workflows/code_quality.yml

@@ -56,7 +56,7 @@ jobs:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
       - name: Pytest + Coverage - ${{ matrix.python-version }}
-        run: uv run --locked pytest -n auto
+        run: uv run --locked pytest -n auto -m "not integration"
 
   generate_docs:
     runs-on: ubuntu-latest

splunk_soar_sdk-3.3.0/.github/workflows/integration_tests.yml

@@ -0,0 +1,195 @@
+name: Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+      - next
+      - beta
+  pull_request:
+
+jobs:
+  build-example-apps:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        app:
+          - example_app
+          - example_app_with_webhook
+    steps:
+      - name: Checkout SDK repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: 3.13
+
+      - name: Build ${{ matrix.app }}
+        run: |
+          echo "Building SOAR SDK from branch"
+          uv build
+          echo "Building ${{ matrix.app }} using soarapps CLI"
+          uv run soarapps package build tests/${{ matrix.app }} --output-file /tmp/${{ matrix.app }}.tgz --with-sdk-wheel-from $(ls -t dist/splunk_soar_sdk-*.whl | head -n 1)
+          echo "App build completed successfully"
+
+      - name: Upload app tar file as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.app }}-tar
+          path: /tmp/${{ matrix.app }}.tgz
+          retention-days: 1
+
+  integration-test:
+    runs-on:
+      - codebuild-github-splunk-soar-sdk-integration-tests-${{ github.run_id }}-${{ github.run_attempt }}
+      - image:custom-linux-875003031410.dkr.ecr.us-west-1.amazonaws.com/soar-connectors/pytest:3ea28df603962918dc8e72ead393c40b3ce6e20e
+    needs: build-example-apps
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - version: "previous"
+            ip: ${{ vars.PHANTOM_INSTANCE_PREVIOUS_VERSION_IP }}
+          - version: "current"
+            ip: ${{ vars.PHANTOM_INSTANCE_CURRENT_VERSION_IP }}
+          - version: "next"
+            ip: ${{ vars.PHANTOM_INSTANCE_NEXT_VERSION_IP }}
+    env:
+      PHANTOM_USERNAME: ${{ vars.PHANTOM_USERNAME }}
+      PHANTOM_PASSWORD: ${{ secrets.PHANTOM_PASSWORD }}
+      NUM_TEST_RETRIES: ${{ vars.NUM_TEST_RETRIES || '2' }}
+      PHANTOM_VERSION: ${{ matrix.version }}
+      AUTOMATION_BROKER_NAME: ${{ vars.AUTOMATION_BROKER_NAME }}
+      FORCE_AUTOMATION_BROKER: ${{ vars.FORCE_AUTOMATION_BROKER }}
+      UV_LOCKED: 1
+    steps:
+      - name: Checkout SDK repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+          python-version: 3.13
+
+      - name: Download example_app tar file
+        uses: actions/download-artifact@v4
+        with:
+          name: example_app-tar
+          path: /tmp/
+
+      - name: Download example_app_with_webhook tar file
+        uses: actions/download-artifact@v4
+        with:
+          name: example_app_with_webhook-tar
+          path: /tmp/
+
+      - name: Clear uv cache and sync SDK
+        run: |
+          uv cache clean
+          uv sync
+
+      - name: Check connectivity to SOAR host
+        run: |
+          apt update -y
+          apt install -y netcat-openbsd
+          phantom_ip=${{ matrix.ip }}
+          nc -zv "$phantom_ip" 443
+          nc -zv "$phantom_ip" 3500
+
+      - name: Install example_app on Phantom instance
+        env:
+          PHANTOM_PASSWORD: ${{ secrets.PHANTOM_PASSWORD }}
+        run: |
+          phantom_ip=${{ matrix.ip }}
+          echo "Installing example_app on https://$phantom_ip"
+
+          uv run soarapps package install \
+            /tmp/example_app.tgz \
+            "$phantom_ip" \
+            --username "${{ vars.PHANTOM_USERNAME }}"
+
+      - name: Install example_app_with_webhook on Phantom instance
+        env:
+          PHANTOM_PASSWORD: ${{ secrets.PHANTOM_PASSWORD }}
+        run: |
+          phantom_ip=${{ matrix.ip }}
+          echo "Installing example_app_with_webhook on https://$phantom_ip"
+
+          uv run soarapps package install \
+            /tmp/example_app_with_webhook.tgz \
+            "$phantom_ip" \
+            --username "${{ vars.PHANTOM_USERNAME }}" \
+            --force
+
+      - name: Run integration tests
+        env:
+          PHANTOM_URL: "https://${{ matrix.ip }}"
+        run: |
+          echo "Running integration tests against $PHANTOM_URL"
+
+          uv sync --group dev
+
+          # Run pytest with integration tests only (no coverage)
+          uv run pytest tests/integration/ \
+            -v \
+            --reruns=$NUM_TEST_RETRIES \
+            --tb=short \
+            --color=yes \
+            --no-cov \
+            -m "integration" \
+            --junitxml=test-results/junit-${{ matrix.version }}.xml \
+            || exit_code=$?
+
+          echo "Integration tests completed"
+          exit ${exit_code:-0}
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: integration-test-results-${{ matrix.version }}
+          path: |
+            pytest-logs/
+            test-results/
+          retention-days: 1
+
+  test-summary:
+    runs-on: ubuntu-latest
+    needs: integration-test
+    if: always()
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Download all test results
+        uses: actions/download-artifact@v4
+        with:
+          pattern: integration-test-results-*
+          path: all-test-results/
+          merge-multiple: true
+
+      - name: Generate test summary
+        run: |
+          RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          python3 .github/scripts/generate_test_summary.py all-test-results/ test-summary.md "$RUN_URL"
+
+      - name: Post summary to job
+        run: |
+          cat test-summary.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload summary as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-summary
+          path: test-summary.md
+          retention-days: 7
+
+      - name: Fail if tests failed
+        run: |
+          if grep -q "FAILED" test-summary.md; then
+            echo "Integration tests failed - check summary above"
+            exit 1
+          fi

{splunk_soar_sdk-3.2.3 → splunk_soar_sdk-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: splunk-soar-sdk
-Version: 3.2.3
+Version: 3.3.0
 Summary: The official framework for developing and testing Splunk SOAR Apps
 Project-URL: Homepage, https://github.com/phantomcyber/splunk-soar-sdk
 Project-URL: Documentation, https://github.com/phantomcyber/splunk-soar-sdk

splunk_soar_sdk-3.3.0/mcp_server/README.md

@@ -0,0 +1,135 @@
+# SOAR Test Assistant MCP Server
+
+An MCP server that provides automated test analysis and remediation capabilities for the SOAR SDK. This server orchestrates test execution, applies targeted fixes, and validates changes through an iterative workflow.
+
+## Workflow
+
+The server implements a two-phase analysis and remediation cycle:
+
+### 1. analyze_tests - Test Execution and Failure Analysis
+Executes the test suite and captures comprehensive diagnostic output including stack traces, assertion failures, and error context.
+
+### 2. fix_and_run_tests - Automated Remediation and Validation
+Applies proposed changes to source files, test fixtures, or dependencies, then re-executes the test suite to verify the fix.
+
+The cycle repeats until all tests pass or unresolvable failures are identified.
+
+## Tools
+
+### `analyze_tests`
+
+Executes the SDK test suite and returns detailed diagnostic output for failure analysis.
+
+**Parameters:**
+- `test_type`: `"unit"` or `"integration"` (default: `"unit"`)
+- `test_path`: Optional specific test file/directory
+- `soar_instance`: Required for integration tests: `{"ip": "...", "username": "...", "password": "..."}`
+
+**Returns:**
+- `status`: `"success"` or `"failed"`
+- `test_output`: Complete pytest output with stack traces and context
+- `message`: Diagnostic guidance for common failure patterns
+
+**Example:**
+```json
+{
+  "test_type": "unit",
+  "test_path": "tests/cli/manifests/test_processors.py"
+}
+```
+
+### `fix_and_run_tests`
+
+Applies a set of proposed changes (file edits, command executions) and re-runs the test suite to validate the remediation.
+
+**Parameters:**
+- `test_type`: `"unit"` or `"integration"` (default: `"unit"`)
+- `test_path`: Optional specific test file/directory
+- `soar_instance`: Required for integration tests
+- `changes`: Array of changes to apply
+
+**Change Types:**
+
+#### edit_file
+Replace exact content in a file:
+```json
+{
+  "type": "edit_file",
+  "file": "tests/example_app/app.json",
+  "old_content": "\"version\": \"1.0\"",
+  "new_content": "\"version\": \"1.1\"",
+  "reasoning": "Update version to match new release"
+}
+```
+
+#### run_command
+Execute a bash command:
+```json
+{
+  "type": "run_command",
+  "command": "cd tests/example_app && uv run python -c \"<regenerate fixture code>\"",
+  "reasoning": "Regenerate outdated test fixture"
+}
+```
+
+**Returns:**
+- `status`: `"success"`, `"still_failing"`, or `"error"`
+- `applied_changes`: List of changes that were applied
+- `test_output`: New test results
+- `errors`: Any errors encountered
+
+## Example Session
+
+Automated remediation workflow for a failing manifest processor test:
+
+1. **Initial Analysis**: `analyze_tests` executes test suite
+   - Detects assertion failure: `app_meta != expected_meta`
+   - Identifies drift in test fixture `tests/example_app/app.json`
+
+2. **Remediation Strategy**: Regenerate test fixture to match current processor output
+   - Command: Invoke `ManifestProcessor` to generate updated manifest
+   - Preserve existing `utctime_updated` field to maintain test stability
+
+3. **Apply and Validate**: `fix_and_run_tests` executes regeneration command
+   - Fixture updated with current expected values
+   - Test suite re-executed
+   - Returns `status: "success"`
+
+## Installation
+
+Run the installation script from the repository root:
+
+```bash
+./install
+```
+
+Or configure manually by adding to your MCP settings (`~/.config/claude-code/mcp_settings.json`):
+
+```json
+{
+  "mcpServers": {
+    "soar-test-assistant": {
+      "command": "uv",
+      "args": ["run", "soar-test-assistant"],
+      "cwd": "/path/to/your/splunk-soar-sdk/mcp_server"
+    }
+  }
+}
+```
+
+Then restart Claude Code.
+
+## SDK-Only Focus
+
+This MCP server is **exclusively for SOAR SDK tests**:
+- **Unit tests**: `tests/` (excluding integration)
+- **Integration tests**: `tests/integration/`
+
+All tests run from the SDK root directory, which is auto-detected.
+
+## Development
+
+```bash
+# Test the MCP server
+uv run pytest
+```

splunk_soar_sdk-3.3.0/mcp_server/install.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -e
+
+echo "Installing SOAR Test Assistant MCP Server..."
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+MCP_CONFIG_DIR="$HOME/.config/claude-code"
+MCP_CONFIG_FILE="$MCP_CONFIG_DIR/mcp_settings.json"
+
+echo "1. Installing dependencies..."
+cd "$SCRIPT_DIR"
+uv sync
+
+echo "2. Setting up MCP configuration..."
+mkdir -p "$MCP_CONFIG_DIR"
+
+SERVER_NAME="soar-test-assistant"
+SERVER_CONFIG=$(cat <<EOF
+{
+  "command": "uv",
+  "args": [
+    "run",
+    "--directory",
+    "$SCRIPT_DIR",
+    "soar_test_assistant"
+  ]
+}
+EOF
+)
+
+if [ -f "$MCP_CONFIG_FILE" ]; then
+    echo "   MCP config file exists, backing up..."
+    cp "$MCP_CONFIG_FILE" "$MCP_CONFIG_FILE.backup.$(date +%s)"
+
+    if command -v jq &> /dev/null; then
+        echo "   Adding server to existing config..."
+        jq ".mcpServers[\"$SERVER_NAME\"] = $SERVER_CONFIG" "$MCP_CONFIG_FILE" > "$MCP_CONFIG_FILE.tmp"
+        mv "$MCP_CONFIG_FILE.tmp" "$MCP_CONFIG_FILE"
+    else
+        echo "   WARNING: jq not found. Please manually add the server to $MCP_CONFIG_FILE"
+        echo "   Server configuration:"
+        echo "$SERVER_CONFIG"
+    fi
+else
+    echo "   Creating new MCP config file..."
+    cat > "$MCP_CONFIG_FILE" <<EOF
+{
+  "mcpServers": {
+    "$SERVER_NAME": $SERVER_CONFIG
+  }
+}
+EOF
+fi
+
+echo ""
+echo "Installation complete!"
+echo ""
+echo "Restart Claude Code to load the new MCP server."

splunk_soar_sdk-3.3.0/mcp_server/mcp_config.json

@@ -0,0 +1,12 @@
+{
+  "mcpServers": {
+    "soar-test-assistant": {
+      "command": "uv",
+      "args": [
+        "run",
+        "soar-test-assistant"
+      ],
+      "cwd": "/path/to/splunk-soar-sdk/mcp_server"
+    }
+  }
+}

splunk_soar_sdk-3.3.0/mcp_server/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "soar-test-assistant"
+version = "0.1.0"
+requires-python = ">=3.13"
+
+dependencies = [
+    "mcp>=1.1.0,<2.0.0",
+    "splunk-soar-sdk>=3.0.0",
+]
+
+[tool.uv.sources]
+splunk-soar-sdk = { path = "..", editable = true }
+
+[project.scripts]
+soar_test_assistant = "soar_test_assistant.server:cli"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "pytest>=7.4.2",
+]

splunk_soar_sdk-3.3.0/mcp_server/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+testpaths = tests

splunk_soar_sdk-3.3.0/mcp_server/src/soar_test_assistant/__init__.py

@@ -0,0 +1,3 @@
+from .server import app, cli
+
+__all__ = ["app", "cli"]