awslabs.dynamodb-mcp-server 1.0.8__tar.gz → 1.0.9__tar.gz

{awslabs_dynamodb_mcp_server-1.0.8 → awslabs_dynamodb_mcp_server-1.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: awslabs.dynamodb-mcp-server
-Version: 1.0.8
+Version: 1.0.9
 Summary: The official MCP Server for interacting with AWS DynamoDB
 Project-URL: homepage, https://awslabs.github.io/mcp/
 Project-URL: docs, https://awslabs.github.io/mcp/servers/dynamodb-mcp-server/
@@ -22,9 +22,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: boto3==1.40.5
+Requires-Dist: dspy-ai>=2.6.27
 Requires-Dist: loguru==0.7.3
 Requires-Dist: mcp[cli]==1.12.4
 Requires-Dist: pydantic==2.11.7
+Requires-Dist: strands-agents>=1.5.0
 Requires-Dist: typing-extensions==4.14.1
 Description-Content-Type: text/markdown
 

{awslabs_dynamodb_mcp_server-1.0.8 → awslabs_dynamodb_mcp_server-1.0.9}/awslabs/dynamodb_mcp_server/__init__.py

@@ -14,4 +14,4 @@
 
 """awslabs.dynamodb-mcp-server"""
 
-__version__ = '1.0.8'
+__version__ = '1.0.9'

{awslabs_dynamodb_mcp_server-1.0.8 → awslabs_dynamodb_mcp_server-1.0.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "awslabs.dynamodb-mcp-server"
-version = "1.0.8"
+version = "1.0.9"
 description = "The official MCP Server for interacting with AWS DynamoDB"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -10,6 +10,8 @@ dependencies = [
     "mcp[cli]==1.12.4",
     "pydantic==2.11.7",
     "typing-extensions==4.14.1",
+    "strands-agents>=1.5.0",
+    "dspy-ai>=2.6.27"
 ]
 license = {text = "Apache-2.0"}
 license-files = ["LICENSE", "NOTICE" ]

awslabs_dynamodb_mcp_server-1.0.9/tests/evals/README.md

@@ -0,0 +1,357 @@
+# DynamoDB MCP Evaluation System
+
+A comprehensive evaluation framework for assessing DynamoDB data modeling guidance quality using advanced conversational AI and structured evaluation methodologies.
+
+## Overview
+
+This evaluation system combines realistic conversational interactions with sophisticated quality assessment to evaluate the effectiveness of DynamoDB modeling guidance. It uses a three-layer architecture integrating Strands agents, MCP protocol, and DSPy evaluation engines to provide objective, systematic assessment of both modeling process quality and technical design excellence.
+
+### Key Features
+
+- **Realistic Conversations**: Uses Strands agents with MCP protocol for authentic user-expert interactions
+- **Dual Evaluation Framework**: Separately assesses modeling process (HOW) and design quality (WHAT)
+- **Expert Knowledge Integration**: Leverages DynamoDB architect prompt for domain-specific evaluation
+- **Comprehensive Scoring**: 10-dimensional assessment covering methodology and technical excellence
+- **Multiple Scenarios**: Predefined scenarios across different complexity levels and domains
+- **Performance Monitoring**: Detailed timing analysis and efficiency metrics
+
+### Dual Evaluation Framework
+
+**Session Evaluation** - Assesses the modeling process quality:
+- Requirements Engineering (1-10)
+- Access Pattern Analysis (1-10)
+- Methodology Adherence (1-10)
+- Technical Reasoning (1-10)
+- Process Documentation (1-10)
+
+**Model Evaluation** - Assesses the technical design quality:
+- Completeness (1-10)
+- Technical Accuracy (1-10)
+- Access Pattern Coverage (1-10)
+- Scalability Considerations (1-10)
+- Cost Optimization (1-10)
+
+## Quick Start
+
+### Prerequisites
+
+1. **AWS Credentials**: Configure AWS access with Bedrock permissions
+```bash
+export AWS_PROFILE=your-profile
+export AWS_REGION=us-east-1
+# OR
+export AWS_ACCESS_KEY_ID=your-key
+export AWS_SECRET_ACCESS_KEY=your-secret
+```
+
+2. **Python Environment**: Python 3.10+ with uv package manager
+```bash
+# Install uv if not already installed
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Navigate to the DynamoDB MCP server directory
+cd src/dynamodb-mcp-server
+```
+
+3. **Dependencies**: Install required packages
+```bash
+uv sync
+```
+
+### Basic Usage
+
+Run a basic evaluation with default settings:
+
+```bash
+uv run python tests/evals/test_dspy_evals.py
+```
+
+This will:
+- Use the default model: `bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0`
+- Run the "Simple E-commerce Schema" scenario
+- Execute the complete evaluation pipeline
+- Display comprehensive results
+
+### Sample Output
+
+```
+🔧 EVALUATION CONFIGURATION
+==============================
+Model: bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0
+Scenario: Simple E-commerce Schema
+
+✅ DSPy configured with bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0
+🎯 Testing scenario complexity: beginner
+🔄 Running conversation for scenario: Simple E-commerce Schema
+...
+✅ Conversation completed in 61.20s
+🔄 Running DSPy session evaluation...
+✅ Session evaluation completed in 11.61s
+📊 Session Score: 8.40 (good)
+🔄 Running DSPy model evaluation...
+✅ Model evaluation completed in 12.12s
+📊 Model Score: 8.20 (good)
+🎯 Complete evaluation finished in 84.93s
+
+============================================================
+COMPREHENSIVE EVALUATION RESULTS
+============================================================
+⏱️  Total Duration: 84.93s
+   • Conversation: 61.20s
+   • Session Evaluation: 11.61s
+   • Model Evaluation: 12.12s
+
+📋 SESSION EVALUATION (Requirements & Methodology)
+--------------------------------------------------
+🎯 Overall Session Score: 8.40 (good)
+
+📊 Detailed Session Scores:
+   • Requirements Engineering: 9.0/10
+   • Access Pattern Analysis: 8.0/10
+   • Methodology Adherence: 8.0/10
+   • Technical Reasoning: 8.0/10
+   • Process Documentation: 9.0/10
+
+🏗️  MODEL EVALUATION (Technical Design)
+--------------------------------------------------
+🎯 Overall Model Score: 8.20 (good)
+
+📊 Detailed Model Scores:
+   • Completeness: 9.0/10
+   • Technical Accuracy: 8.0/10
+   • Access Pattern Coverage: 9.0/10
+   • Scalability Considerations: 8.0/10
+   • Cost Optimization: 7.0/10
+
+🎖️  QUALITY SUMMARY
+--------------------------------------------------
+Session Quality: good
+Model Quality: good
+```
+
+## Command Line Usage
+
+### Available Commands
+
+**Basic evaluation:**
+```bash
+uv run python tests/evals/test_dspy_evals.py
+```
+
+**Custom model evaluation:**
+```bash
+uv run python tests/evals/test_dspy_evals.py --model "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
+```
+
+**Specific scenario testing:**
+```bash
+uv run python tests/evals/test_dspy_evals.py --scenario "High-Scale Social Media Platform"
+```
+
+**Combined configuration:**
+```bash
+uv run python tests/evals/test_dspy_evals.py --model "custom-model" --scenario "Content Management System"
+```
+
+**List available scenarios:**
+```bash
+uv run python tests/evals/test_dspy_evals.py --list-scenarios
+```
+
+### Command Line Options
+
+| Option | Description | Default |
+|--------|-------------|---------|
+| `--model` | Bedrock model ID to use | `bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0` |
+| `--scenario` | Scenario name to evaluate | `"Simple E-commerce Schema"` |
+| `--list-scenarios` | Show all available scenarios | - |
+| `--debug` | Show raw JSON output | - |
+| `--aws-profile` | AWS profile to use for evaluation | `Bedrock` |
+
+## Available Scenarios
+
+The system includes predefined scenarios across different complexity levels:
+
+### Beginner Scenarios
+- **Simple E-commerce Schema**: Basic online retail with users, products, orders
+- **Content Management System**: Blog/CMS with articles, authors, categories
+
+### Advanced Scenarios
+- **High-Scale Social Media Platform**: Social media with posts, likes, comments at scale
+
+### Scenario Structure
+
+Each scenario includes:
+- **Application Details**: Type, domain, business model
+- **Entities & Relationships**: Complete data model definition
+- **Access Patterns**: Read/write patterns with performance requirements
+- **Scale Requirements**: User base, transaction volume, growth projections
+- **Performance Targets**: Latency and throughput specifications
+
+To see all scenarios with descriptions:
+```bash
+uv run python tests/evals/test_dspy_evals.py --list-scenarios
+```
+
+## Understanding Results
+
+### Quality Levels
+
+Results are classified into quality levels based on overall scores:
+
+| Score Range | Quality Level | Description |
+|-------------|---------------|-------------|
+| 8.5 - 10.0 | `excellent` | Exceptional quality - ready for production |
+| 7.0 - 8.4 | `good` | Solid quality - minor improvements needed |
+| 5.5 - 6.9 | `acceptable` | Adequate - meets basic requirements |
+| 4.0 - 5.4 | `needs_improvement` | Deficient - significant gaps present |
+| 1.0 - 3.9 | `poor` | Major issues - substantial rework required |
+
+### Performance Characteristics
+
+Typical evaluation timing:
+- **Conversation Phase**: 30-60 seconds (depends on model and scenario complexity)
+- **Session Evaluation**: 10-15 seconds (DSPy process assessment)
+- **Model Evaluation**: 10-15 seconds (DSPy design assessment)
+- **Total Duration**: 50-90 seconds for complete pipeline
+
+### Session vs Model Evaluation
+
+**Session Evaluation** focuses on **HOW** the modeling was conducted:
+- Did the system follow proper methodology?
+- Were requirements properly gathered and analyzed?
+- Was the decision-making process well-documented?
+- Were trade-offs and alternatives considered?
+
+**Model Evaluation** focuses on **WHAT** was delivered:
+- Is the final design technically correct?
+- Does it handle all required access patterns?
+- Are scalability concerns addressed?
+- Is the solution cost-optimized?
+
+## Configuration
+
+### Model Selection
+
+The system supports any Bedrock-compatible model. Popular choices:
+
+```bash
+# Claude 4 Sonnet (recommended)
+--model "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+
+# Claude 3.5 Sonnet
+--model "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
+
+# Other Bedrock models
+--model "bedrock/amazon.titan-text-premier-v1:0"
+```
+
+### Environment Variables
+
+| Variable | Purpose | Default |
+|----------|---------|---------|
+| `AWS_PROFILE` | AWS credential profile | - |
+| `AWS_REGION` | AWS region for Bedrock | `us-east-1` |
+| `AWS_ACCESS_KEY_ID` | Direct AWS credentials | - |
+| `AWS_SECRET_ACCESS_KEY` | Direct AWS credentials | - |
+
+## Troubleshooting
+
+### Common Issues
+
+**AWS Credentials Error:**
+```
+AWS credentials not available - set AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY or AWS_PROFILE
+```
+**Solution**: Configure AWS credentials as shown in Prerequisites section.
+
+**Model Access Error:**
+```
+Could not access the model: bedrock/model-name
+```
+**Solution**: Ensure your AWS account has access to the requested Bedrock model and proper permissions.
+
+**Scenario Not Found:**
+```
+Scenario 'Invalid Name' not found
+```
+**Solution**: Use `--list-scenarios` to see available options and check spelling.
+
+**MCP Connection Issues:**
+```
+Error during Strands conversation: MCP connection failed
+```
+**Solution**: Ensure the DynamoDB MCP server is properly installed and accessible.
+
+### Extending the System
+
+**Adding New Scenarios:**
+1. Add scenario definition to `scenarios.py`
+2. Include all required fields (entities, access patterns, scale)
+3. Test with different models for consistency
+
+**Adding New Evaluation Dimensions:**
+
+The system uses a dynamic registry pattern that makes adding new evaluation dimensions extremely simple - no code changes required to the evaluation engine!
+
+**3-Step Process:**
+1. **Add Dimension to Registry** - Simply add a new `DimensionConfig` to the appropriate evaluation in `evaluation_registry.py`
+2. **Define Scoring Rubric** - Specify how the dimension should be evaluated (1-10 scale)
+3. **Test Immediately** - The dynamic evaluator automatically picks up the new dimension
+
+**Example - Adding Security Evaluation:**
+```python
+# In evaluation_registry.py, add to model_dimensions list:
+DimensionConfig(
+    name="security_considerations",
+    display_name="Security Considerations",
+    description="Data security and access control planning",
+    scoring_rubric=(
+        "Score 1-10: Evaluate security measures including "
+        "encryption, access patterns, IAM policies, and data protection. "
+        "Return single number 1-10."
+    ),
+    weight=1.0,
+    justification_prompt="Explain security assessment and recommendations"
+)
+```
+
+**That's it!** The `DynamicEvaluationEngine` will automatically:
+- Generate DSPy signatures with your new dimension
+- Create result dataclasses including the new field
+- Integrate scoring and justification collection
+- Make it available in CLI evaluations
+
+**No changes needed to:**
+- `dynamic_evaluators.py` - automatically adapts
+- `test_dspy_evals.py` - CLI works immediately
+- Result processing - handled automatically
+
+**New Model Support:**
+1. Ensure model is available in AWS Bedrock
+2. Test compatibility with DSPy framework
+3. Adjust timeout settings if needed
+
+**Architecture Overview:**
+- `evaluation_registry.py`: Dynamic registry for evaluation dimensions and types
+- `dynamic_evaluators.py`: DSPy evaluation engine that adapts to registry configurations
+- `multiturn_evaluator.py`: Multi-turn conversation evaluator using Strands agents
+- `scenarios.py`: Test scenario definitions for evaluation
+- `test_dspy_evals.py`: Command-line interface for the evaluation system
+
+## Development and Contributing
+
+### Running Tests
+
+```bash
+# Run a quick evaluation
+uv run python tests/evals/test_dspy_evals.py
+
+# Test different models
+uv run python tests/evals/test_dspy_evals.py --model "bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
+
+# Test all scenarios
+for scenario in "Simple E-commerce Schema" "High-Scale Social Media Platform" "Content Management System"; do
+  uv run python tests/evals/test_dspy_evals.py --scenario "$scenario"
+done
+```

awslabs_dynamodb_mcp_server-1.0.9/tests/evals/dynamic_evaluators.py

@@ -0,0 +1,251 @@
+"""Dynamic DSPy evaluator using evaluation registry for easy dimension management."""
+
+import dspy
+import json
+from dataclasses import dataclass
+from evaluation_registry import EvaluationConfig, registry
+from logging_config import get_logger
+from pathlib import Path
+from typing import Any, Dict, Type
+
+
+# Initialize logger for this module
+logger = get_logger(__name__)
+
+
+def create_dspy_signature(evaluation_config: EvaluationConfig) -> Type[dspy.Signature]:
+    """Dynamically create a DSPy signature class based on evaluation configuration."""
+    signature_attrs = {}
+
+    input_fields = evaluation_config.input_fields or {}
+    for field_name, field_desc in input_fields.items():
+        signature_attrs[field_name] = dspy.InputField(desc=field_desc)
+
+    for dimension in evaluation_config.dimensions:
+        score_field_name = f'{dimension.name}_score'
+        signature_attrs[score_field_name] = dspy.OutputField(desc=dimension.scoring_rubric)
+
+    for dimension in evaluation_config.dimensions:
+        if dimension.justification_prompt:
+            justification_field_name = f'{dimension.name}_justification'
+            signature_attrs[justification_field_name] = dspy.OutputField(
+                desc=dimension.justification_prompt
+            )
+
+    signature_attrs['strengths'] = dspy.OutputField(
+        desc=f'Key strengths of the {evaluation_config.display_name.lower()}, highlighting what was done exceptionally well'
+    )
+    signature_attrs['weaknesses'] = dspy.OutputField(
+        desc=f'Main weaknesses and areas where the {evaluation_config.display_name.lower()} fell short or could be significantly improved'
+    )
+    signature_attrs['improvement_recommendations'] = dspy.OutputField(
+        desc=f'Specific, actionable recommendations for improving the {evaluation_config.display_name.lower()}, with concrete suggestions for addressing identified weaknesses'
+    )
+
+    signature_class_name = f'{evaluation_config.name.title().replace("_", "")}Signature'
+    signature_class = type(signature_class_name, (dspy.Signature,), signature_attrs)
+
+    signature_class.__doc__ = f'Generated DSPy signature for {evaluation_config.display_name} with {len(evaluation_config.dimensions)} dimensions.'
+
+    return signature_class
+
+
+def create_result_dataclass(evaluation_config: EvaluationConfig) -> Type:
+    """Dynamically create a result dataclass based on evaluation configuration."""
+    class_fields = []
+
+    for dimension in evaluation_config.dimensions:
+        class_fields.append((dimension.name, float))
+
+    class_fields.extend(
+        [('justifications', Dict[str, str]), ('overall_score', float), ('quality_level', str)]
+    )
+
+    dataclass_name = f'{evaluation_config.name.title().replace("_", "")}Result'
+    annotations = dict(class_fields)
+    result_class = type(dataclass_name, (), {'__annotations__': annotations})
+    result_class = dataclass(result_class)
+
+    def to_dict(self):
+        """Convert result object to dictionary for JSON serialization."""
+        result_dict = {}
+        for dimension in evaluation_config.dimensions:
+            result_dict[dimension.name] = getattr(self, dimension.name)
+        result_dict.update(
+            {
+                'justifications': self.justifications,
+                'overall_score': self.overall_score,
+                'quality_level': self.quality_level,
+            }
+        )
+        return result_dict
+
+    setattr(result_class, 'to_dict', to_dict)
+    result_class.__doc__ = f'Generated result container for {evaluation_config.display_name} with {len(evaluation_config.dimensions)} dimensions.'
+
+    return result_class
+
+
+class DynamicEvaluationEngine:
+    """Dynamic evaluation engine that adapts to any registered evaluation type."""
+
+    def __init__(self):
+        """Initialize the dynamic evaluation engine with empty caches."""
+        self._evaluators = {}
+        self._result_classes = {}
+        self._expert_knowledge_cache = None
+
+    def _get_evaluator(self, evaluation_name: str):
+        """Get or create evaluator for the specified evaluation type."""
+        if evaluation_name not in self._evaluators:
+            evaluation_config = registry.get_evaluation(evaluation_name)
+            signature_class = create_dspy_signature(evaluation_config)
+            self._evaluators[evaluation_name] = dspy.ChainOfThought(signature_class)
+
+        return self._evaluators[evaluation_name]
+
+    def _get_result_class(self, evaluation_name: str):
+        """Get or create result class for the specified evaluation type."""
+        if evaluation_name not in self._result_classes:
+            evaluation_config = registry.get_evaluation(evaluation_name)
+            self._result_classes[evaluation_name] = create_result_dataclass(evaluation_config)
+
+        return self._result_classes[evaluation_name]
+
+    def _load_expert_knowledge(self) -> str:
+        """Load DynamoDB expert knowledge (cached)."""
+        if self._expert_knowledge_cache is None:
+            try:
+                prompt_path = (
+                    Path(__file__).parent.parent.parent
+                    / 'awslabs'
+                    / 'dynamodb_mcp_server'
+                    / 'prompts'
+                    / 'dynamodb_architect.md'
+                )
+                self._expert_knowledge_cache = prompt_path.read_text(encoding='utf-8')
+            except Exception as e:
+                logger.warning(f'Warning: Could not load expert knowledge: {e}')
+                self._expert_knowledge_cache = 'Expert knowledge not available.'
+
+        return self._expert_knowledge_cache
+
+    def evaluate(self, evaluation_name: str, scenario: Dict[str, Any], content: str, **kwargs):
+        """Evaluate content using the specified evaluation type."""
+        evaluation_config = registry.get_evaluation(evaluation_name)
+        evaluator = self._get_evaluator(evaluation_name)
+        result_class = self._get_result_class(evaluation_name)
+
+        # Prepare input arguments
+        eval_inputs = {}
+
+        scenario_json = json.dumps(scenario, indent=2)
+
+        input_mappings = {
+            'scenario_requirements': scenario_json,
+            'guidance_response': content,
+            'modeling_requirement_content': content,
+            'dynamodb_expert_knowledge': self._load_expert_knowledge(),
+            'architect_methodology': self._load_expert_knowledge(),
+        }
+
+        # Add inputs based on evaluation configuration
+        for field_name in evaluation_config.input_fields.keys():
+            if field_name in input_mappings:
+                eval_inputs[field_name] = input_mappings[field_name]
+            elif field_name in kwargs:
+                eval_inputs[field_name] = kwargs[field_name]
+
+        # Run the evaluation
+        raw_result = evaluator(**eval_inputs)
+
+        # Process results into structured format
+        return self._process_results(evaluation_config, raw_result, result_class)
+
+    def _process_results(self, evaluation_config: EvaluationConfig, raw_result, result_class):
+        """Process raw DSPy results into structured result object."""
+        # Extract dimension scores
+        dimension_scores = {}
+        for dimension in evaluation_config.dimensions:
+            score_field = f'{dimension.name}_score'
+            score_value = getattr(raw_result, score_field, 0.0)
+            # Handle DSPy returning various types
+            if isinstance(score_value, (int, float)):
+                dimension_scores[dimension.name] = float(score_value)
+            else:
+                # Try to parse if string
+                try:
+                    dimension_scores[dimension.name] = float(str(score_value).split()[0])
+                except (ValueError, IndexError):
+                    dimension_scores[dimension.name] = 0.0
+
+        # Calculate overall score using weighted average
+        total_weight = sum(dim.weight for dim in evaluation_config.dimensions)
+        if total_weight > 0:
+            weighted_sum = sum(
+                dimension_scores[dim.name] * dim.weight for dim in evaluation_config.dimensions
+            )
+            overall_score = round(weighted_sum / total_weight, 2)
+        else:
+            overall_score = 0.0
+
+        # Determine quality level using existing thresholds
+        quality_thresholds = {
+            'excellent': 8.5,
+            'good': 7.0,
+            'acceptable': 5.5,
+            'needs_improvement': 4.0,
+            'poor': 2.0,
+        }
+
+        if overall_score >= quality_thresholds['excellent']:
+            quality_level = 'excellent'
+        elif overall_score >= quality_thresholds['good']:
+            quality_level = 'good'
+        elif overall_score >= quality_thresholds['acceptable']:
+            quality_level = 'acceptable'
+        elif overall_score >= quality_thresholds['needs_improvement']:
+            quality_level = 'needs_improvement'
+        else:
+            quality_level = 'poor'
+
+        # Build justifications dictionary
+        justifications = {}
+
+        # Add dimension justifications
+        for dimension in evaluation_config.dimensions:
+            if dimension.justification_prompt:
+                justification_field = f'{dimension.name}_justification'
+                justifications[dimension.name] = str(getattr(raw_result, justification_field, ''))
+
+        # Add overall assessment fields
+        justifications.update(
+            {
+                'strengths': str(getattr(raw_result, 'strengths', '')),
+                'weaknesses': str(getattr(raw_result, 'weaknesses', '')),
+                'improvement_recommendations': str(
+                    getattr(raw_result, 'improvement_recommendations', '')
+                ),
+            }
+        )
+
+        # Create result object
+        result_kwargs = {
+            **dimension_scores,
+            'justifications': justifications,
+            'overall_score': overall_score,
+            'quality_level': quality_level,
+        }
+
+        return result_class(**result_kwargs)
+
+
+# Create global instance
+dynamic_engine = DynamicEvaluationEngine()
+
+__all__ = [
+    'DynamicEvaluationEngine',
+    'create_dspy_signature',
+    'create_result_dataclass',
+    'dynamic_engine',
+]