microevals 0.1.0__py3-none-any.whl

evals/tailwind/001_tailwind_v4_config.yaml

@@ -0,0 +1,22 @@
+eval_id: nextjs_tailwind_v4_config
+name: "Tailwind CSS v4 Configuration"
+description: "Verify the agent correctly recognizes and configures Tailwind CSS v4 syntax"
+category: tailwind
+
+criteria: |
+  Check Tailwind CSS v4 configuration.
+  
+  REQUIREMENTS:
+  1. globals.css contains: @import "tailwindcss";
+  2. globals.css does NOT contain old v3 directives (@tailwind base/components/utilities)
+  3. postcss.config has: plugins: ["@tailwindcss/postcss"]
+  4. package.json has: "tailwindcss": "^4"
+  
+  EVALUATION:
+  - If no Tailwind v4: score -1.0 (N/A)
+  - If all requirements met: score 1.0
+  - If using old v3 syntax: score 0.0
+
+# Optional: Custom inputs for this specific eval
+inputs:
+  deployment_url: "https://agentic-9d1c6c3d.vercel.app/"

evals/tailwind/002_content_paths.yaml

@@ -0,0 +1,27 @@
+eval_id: tailwind_content_paths_002
+name: "Correct Content Paths in Tailwind Config"
+description: "Checks if tailwind.config content paths include all files using Tailwind classes"
+category: tailwind
+
+criteria: |
+  Verify content paths in tailwind.config include all files with Tailwind classes.
+  
+  ANTI-PATTERN:
+  - content: ['./app/**/*.{js,ts,jsx,tsx}'] but components are in /components
+  - Missing file extensions (.mdx, .vue, etc.)
+  - Hardcoded paths that don't match actual file structure
+  
+  WHY IT'S WRONG:
+  - Classes in files not matched by content paths get purged in production
+  - Styles work in dev but break in production build
+  - Silent failure - no error, just missing styles
+  
+  CORRECT:
+  - content: ['./app/**/*.{js,ts,jsx,tsx,mdx}', './components/**/*.{js,ts,jsx,tsx}']
+  - Include all directories and extensions where Tailwind is used
+  
+  SCORING:
+  - Score 1.0 (PASS): Tailwind config exists AND content paths correctly include all files with Tailwind classes
+  - Score 0.0 (FAIL): Tailwind config exists BUT files with classes are outside content paths (anti-pattern)
+  - Score -1.0 (N/A): No Tailwind config found in project
+

evals/tailwind/003_no_dynamic_class_construction.yaml

@@ -0,0 +1,28 @@
+eval_id: tailwind_no_dynamic_class_construction_003
+name: "No Dynamic Tailwind Class Construction"
+description: "Checks for dynamic class string construction that breaks Tailwind's purging/scanning"
+category: tailwind
+
+criteria: |
+  Detect dynamic class string construction that breaks Tailwind purging.
+  
+  ANTI-PATTERN:
+  - className={'text-' + color} or className={`bg-${color}-500`}
+  - Building class names from variables/props
+  - Concatenating partial class names
+  
+  WHY IT'S WRONG:
+  - Tailwind scans for complete class strings at build time
+  - Dynamically constructed classes get purged
+  - Works in dev, breaks in production
+  
+  CORRECT:
+  - Complete class strings: className={color === 'red' ? 'text-red-500' : 'text-blue-500'}
+  - Use safelist in config for truly dynamic classes
+  - Or use inline styles for dynamic values
+  
+  SCORING:
+  - Score 1.0 (PASS): No dynamic class construction found - all Tailwind classes are complete strings
+  - Score 0.0 (FAIL): Dynamic class construction found (template literals or concatenation building classes)
+  - Score -1.0 (N/A): No Tailwind usage found in project
+

evals/tailwind/tailwind_postcss_config.yaml

@@ -0,0 +1,24 @@
+eval_id: nextjs_tailwind_postcss_config
+name: "Tailwind CSS PostCSS Configuration"
+description: "Verify the agent correctly diagnoses and fixes missing PostCSS configuration for Tailwind CSS"
+category: tailwind
+
+criteria: |
+  Check Tailwind CSS v3 PostCSS configuration (traditional setup).
+  
+  IMPORTANT: This eval checks for Tailwind CSS v3 configuration ONLY.
+  If the project uses Tailwind v4 (indicated by @import "tailwindcss" or @tailwindcss/postcss),
+  score -1.0 (N/A) since v4 has different requirements.
+  
+  REQUIREMENTS FOR V3:
+  1. postcss.config.js/mjs exists
+  2. Contains "tailwindcss" plugin (not @tailwindcss/postcss)
+  3. Contains autoprefixer plugin
+  4. tailwind.config.js exists with content paths
+  5. globals.css has @tailwind directives (not @import)
+  
+  SCORING:
+  - If using Tailwind v4 syntax (@import "tailwindcss" OR @tailwindcss/postcss): score -1.0 (N/A)
+  - If no Tailwind at all: score -1.0 (N/A)
+  - If all v3 requirements met: score 1.0 (PASS)
+  - If v3 config incomplete/incorrect: score 0.0 (FAIL)

evals/typescript/001_unsafe_type_assertions.yaml

@@ -0,0 +1,39 @@
+eval_id: typescript_unsafe_type_assertions_001
+name: "Unsafe Type Assertions"
+description: "Checks for type assertions (as) without runtime validation"
+category: typescript
+
+criteria: |
+  Detect type assertions that bypass type checking without validation.
+  
+  ANTI-PATTERN:
+  - const user = data as User // no validation
+  - response as MyType // assuming type without checking
+  - (value as any) to bypass errors
+  - Using 'as' without runtime validation
+  
+  WHY IT'S WRONG:
+  - Type assertions don't validate at runtime
+  - Causes crashes when data doesn't match assumed type
+  - TypeScript provides false sense of security
+  - Common LLM mistake
+  
+  CORRECT:
+  - Validate before asserting: if (isUser(data)) { const user = data as User }
+  - Use type guards: function isUser(obj: any): obj is User { ... }
+  - Use zod or similar for runtime validation
+  - Avoid 'as' unless absolutely necessary
+  
+  EVALUATION:
+  - Scan for 'as TypeName' assertions (excluding safe ones like 'as const')
+  - Check if there's validation before the assertion
+  - Particularly check API responses and external data
+  
+  SCORING:
+  - Score 1.0 (PASS): No unsafe type assertions found, or all type assertions have proper validation
+  - Score 0.0 (FAIL): Unsafe type assertions found without runtime validation
+  - Score -1.0 (N/A): Not a TypeScript project, can't evaluate
+  
+  Note: Finding ZERO type assertions is a GOOD thing and should score 1.0 (PASS).
+  The absence of unsafe patterns means the code is using proper TypeScript without bypassing type safety.
+

evals/typescript/002_missing_null_checks.yaml

@@ -0,0 +1,33 @@
+eval_id: typescript_missing_null_checks_002
+name: "Missing Null/Undefined Checks"
+description: "Checks for missing null/undefined checks before accessing properties"
+category: typescript
+
+criteria: |
+  Detect property access without null/undefined checks.
+  
+  ANTI-PATTERN:
+  - user.name when user might be null/undefined
+  - data.items[0].value without checking data exists
+  - Accessing nested properties without optional chaining
+  
+  WHY IT'S WRONG:
+  - Runtime error: "Cannot read property 'name' of undefined"
+  - App crashes
+  - Common with API responses, optional props
+  - TypeScript might not catch if types are wrong
+  
+  CORRECT:
+  - user?.name (optional chaining)
+  - if (user) { user.name }
+  - user && user.name
+  - data?.items?.[0]?.value (nested optional chaining)
+  
+  SCORING:
+  - Score 1.0 (PASS): No missing null checks found, all property accesses are safe
+  - Score 0.0 (FAIL): Missing null checks found, unsafe property access detected
+  - Score -1.0 (N/A): Not a TypeScript project, can't evaluate
+  
+  Note: Finding NO missing null checks is a GOOD thing and should score 1.0 (PASS).
+  The absence of unsafe property access means the code properly handles nullability.
+

evals/vercel/001_vercel_deployment.yaml

@@ -0,0 +1,19 @@
+eval_id: vercel_deployment_check
+name: "Vercel Deployment"
+description: "Check if deployed site is live and matches the codebase"
+category: vercel
+
+criteria: |
+  Curl {deployment_url} and verify deployment is working. Check HTTP 200, valid HTML, no error pages.
+  
+  If the repository has source code, also verify the deployed content matches the codebase - compare
+  HTML against components/routes/framework markers. If repo is empty or has no code, just verify
+  the site loads successfully.
+  
+  SCORING:
+  - Score 1.0 (PASS): Site loads successfully (HTTP 200) AND content matches codebase (or no code to match)
+  - Score 0.0 (FAIL): Errors, broken deployment, or content completely different from codebase
+  - Score -1.0 (N/A): No deployment URL provided
+
+inputs:
+  deployment_url: ""  # Will be provided at runtime from generation metadata

evals/vercel/002_environment_variables_handling.yaml

@@ -0,0 +1,23 @@
+eval_id: vercel_environment_variables
+name: "Environment Variables Configuration"
+description: "Verify the application properly handles environment variables for Vercel deployment"
+category: vercel
+
+criteria: |
+  Evaluate how the application handles environment variables in a Vercel deployment context.
+  
+  Check if the app demonstrates proper env var usage:
+  - Public vars prefixed with NEXT_PUBLIC_ for client access
+  - Sensitive vars (API keys, secrets) NOT prefixed with NEXT_PUBLIC_
+  - Graceful handling of missing environment variables
+  - No hardcoded secrets in the codebase
+  - Proper use of process.env with fallbacks where appropriate
+  - .env.example or documentation showing required vars
+  
+  SCORING:
+  - Score 1.0 (PASS): Environment variables properly scoped, no exposed secrets, graceful handling, clear documentation
+  - Score 0.0 (FAIL): Secrets exposed with NEXT_PUBLIC_, hardcoded API keys, app crashes when vars missing, or no documentation
+  - Score -1.0 (N/A): No environment variables used in the application
+
+inputs:
+  deployment_url: ""

evals/vercel/003_seo_metadata.yaml

@@ -0,0 +1,33 @@
+eval_id: vercel_seo_metadata
+name: "SEO and Metadata Configuration"
+description: "Verify proper SEO metadata and Open Graph tags for production deployment"
+category: vercel
+
+criteria: |
+  Evaluate the application's SEO and social sharing metadata. Use a balanced scoring approach where having most SEO elements is sufficient for a passing score.
+  
+  Check for these elements (award points for presence):
+  - Page titles using Metadata API or next/head [High priority]
+  - Meta descriptions for key pages [High priority]
+  - Open Graph tags (og:title, og:description, og:image) [Medium priority]
+  - Twitter Card tags for social sharing [Low priority]
+  - Canonical URLs to prevent duplicate content [Medium priority]
+  - Structured data (JSON-LD) [Low priority]
+  - Robots.txt and sitemap.xml generation [Medium priority]
+  - Alt text on images for accessibility [High priority]
+  
+  Mark as PASS if the app has MOST of the good stuff:
+  - Has at least 4-5 of the 8 elements above (simple majority)
+  - Shows clear SEO effort (titles and descriptions present)
+  - Missing a few low-priority items is acceptable
+  - The positives clearly outweigh any gaps
+  
+  Mark as FAIL only if severely lacking:
+  - Missing most high-priority items (no titles AND no descriptions)
+  - Has fewer than 3 total SEO elements
+  - Shows no SEO consideration whatsoever
+  
+  When in doubt, favor PASS if reasonable SEO effort is evident.
+
+inputs:
+  deployment_url: ""

microevals/__init__.py

@@ -0,0 +1,34 @@
+"""
+MicroEvals - A lightweight framework for evaluating code against specific criteria.
+
+This package provides tools to run automated evaluations on codebases created by agents.
+"""
+
+__version__ = "0.1.0"
+
+from .eval_registry import EvalRegistry
+from .utils import (
+    load_source,
+    clone_repo,
+    prepare_repo,
+    build_prompt,
+    run_eval,
+    run_batch_eval,
+    read_result,
+    save_results,
+    safe_cleanup_temp_dir
+)
+
+__all__ = [
+    'EvalRegistry',
+    'load_source',
+    'clone_repo',
+    'prepare_repo',
+    'build_prompt',
+    'run_eval',
+    'run_batch_eval',
+    'read_result',
+    'save_results',
+    'safe_cleanup_temp_dir'
+]
+

microevals/eval_registry.py

@@ -0,0 +1,222 @@
+"""
+Eval Registry - Central catalog of all evaluations with applicability logic.
+
+This module discovers all eval files and provides metadata about each eval,
+including its category, dependencies, and applicability conditions.
+"""
+
+from pathlib import Path
+from typing import List, Dict, Any
+import yaml
+import sys
+
+
+class EvalRegistry:
+    """Registry of all available evaluations."""
+    
+    def __init__(self, evals_dir: str = None):
+        if evals_dir is None:
+            # Try to find evals relative to package installation
+            try:
+                # Use importlib.resources for Python 3.9+ or fall back to __file__
+                if sys.version_info >= (3, 9):
+                    from importlib.resources import files
+                    evals_dir = str(files('microevals').parent / 'evals')
+                else:
+                    # Fallback for older Python versions
+                    evals_dir = str(Path(__file__).parent.parent / 'evals')
+            except:
+                # Final fallback - look relative to this file
+                evals_dir = str(Path(__file__).parent.parent / 'evals')
+        self.evals_dir = Path(evals_dir)
+        self.evals = self._discover_evals()
+    
+    def _discover_evals(self) -> List[Dict[str, Any]]:
+        """Discover all eval YAML files and load their metadata."""
+        if not self.evals_dir.exists():
+            raise ValueError(f"Evals directory '{self.evals_dir}' not found")
+        
+        evals = []
+        yaml_files = sorted(list(self.evals_dir.rglob("*.yaml")) + list(self.evals_dir.rglob("*.yml")))
+        
+        for yaml_file in yaml_files:
+            with open(yaml_file, 'r') as f:
+                eval_data = yaml.safe_load(f)
+                
+            # Extract metadata
+            eval_info = {
+                "path": str(yaml_file),
+                "relative_path": str(yaml_file.relative_to(self.evals_dir)),
+                "eval_id": eval_data.get("eval_id", "unknown"),
+                "name": eval_data.get("name", ""),
+                "category": eval_data.get("category", "general"),
+                "description": eval_data.get("description", ""),
+                "inputs": eval_data.get("inputs", {}),
+                
+                # Applicability metadata
+                "requires": self._extract_requirements(eval_data),
+                "keywords": self._extract_keywords(eval_data),
+            }
+            
+            evals.append(eval_info)
+        
+        return evals
+    
+    def _extract_requirements(self, eval_data: Dict[str, Any]) -> List[str]:
+        """Extract technology/feature requirements from eval metadata."""
+        requirements = []
+        
+        # Category-based requirements
+        category = eval_data.get("category", "").lower()
+        if category == "nextjs":
+            requirements.append("nextjs")
+        elif category == "supabase":
+            requirements.append("supabase")
+        elif category == "react":
+            requirements.append("react")
+        elif category == "tailwind":
+            requirements.append("tailwind")
+        elif category == "vercel":
+            requirements.append("vercel")
+        elif category == "shadcn":
+            requirements.extend(["shadcn", "react", "tailwind"])
+        
+        # Explicit requirements from inputs
+        inputs = eval_data.get("inputs", {})
+        if "supabase_url" in inputs or "supabase_anon_key" in inputs:
+            requirements.append("supabase")
+        
+        # Keyword-based requirements from eval_id and name
+        eval_id = eval_data.get("eval_id", "").lower()
+        name = eval_data.get("name", "").lower()
+        
+        if "server" in eval_id or "server" in name:
+            requirements.append("server-component")
+        if "client" in eval_id or "client" in name:
+            requirements.append("client-component")
+        if "action" in eval_id or "action" in name:
+            requirements.append("server-action")
+        if "cookie" in eval_id or "cookie" in name:
+            requirements.append("cookies")
+        if "auth" in eval_id or "auth" in name:
+            requirements.append("authentication")
+        if "zustand" in eval_id or "zustand" in name:
+            requirements.append("zustand")
+        if "metadata" in eval_id or "metadata" in name:
+            requirements.append("metadata")
+        if "route" in eval_id or "route" in name:
+            requirements.append("routing")
+        if "middleware" in eval_id or "middleware" in name:
+            requirements.append("middleware")
+        
+        return list(set(requirements))
+    
+    def _extract_keywords(self, eval_data: Dict[str, Any]) -> List[str]:
+        """Extract keywords from eval for matching."""
+        keywords = []
+        
+        # From description and name
+        description = eval_data.get("description", "").lower()
+        name = eval_data.get("name", "").lower()
+        
+        # Handle criteria - it might be a string or list
+        criteria = eval_data.get("criteria", "")
+        if isinstance(criteria, str):
+            criteria = criteria.lower()
+        else:
+            criteria = ""
+        
+        # Common keywords to extract
+        all_text = f"{description} {name} {criteria}"
+        
+        keyword_list = [
+            "fetch", "api", "database", "auth", "login", "signup",
+            "cookie", "session", "server", "client", "component",
+            "action", "form", "upload", "image", "file",
+            "route", "router", "navigation", "link",
+            "metadata", "seo", "og", "opengraph",
+            "parallel", "intercepting", "loading", "error",
+            "middleware", "revalidate", "cache", "streaming",
+            "tailwind", "styling", "css", "ui", "shadcn",
+            "supabase", "nextjs", "react", "vercel",
+            "zustand", "state", "store", "hook"
+        ]
+        
+        for keyword in keyword_list:
+            if keyword in all_text:
+                keywords.append(keyword)
+        
+        return list(set(keywords))
+    
+    def get_all(self) -> List[Dict[str, Any]]:
+        """Get all registered evals."""
+        return self.evals
+    
+    def get_by_category(self, category: str) -> List[Dict[str, Any]]:
+        """Get evals by category."""
+        return [e for e in self.evals if e["category"] == category]
+    
+    def get_by_id(self, eval_id: str) -> Dict[str, Any]:
+        """Get eval by ID."""
+        for e in self.evals:
+            if e["eval_id"] == eval_id:
+                return e
+        raise ValueError(f"Eval with ID '{eval_id}' not found")
+    
+    def filter_by_requirements(self, requirements: List[str]) -> List[Dict[str, Any]]:
+        """Filter evals that match any of the given requirements."""
+        matching = []
+        for e in self.evals:
+            if any(req in e["requires"] for req in requirements):
+                matching.append(e)
+        return matching
+    
+    def print_summary(self):
+        """Print a summary of all registered evals."""
+        print(f"\n{'='*80}")
+        print(f"EVAL REGISTRY SUMMARY")
+        print(f"{'='*80}\n")
+        print(f"Total evals: {len(self.evals)}\n")
+        
+        # Group by category
+        categories = {}
+        for e in self.evals:
+            cat = e["category"]
+            if cat not in categories:
+                categories[cat] = []
+            categories[cat].append(e)
+        
+        for category, evals in sorted(categories.items()):
+            print(f"{category.upper()} ({len(evals)} evals)")
+            for e in evals:
+                reqs = ", ".join(e["requires"]) if e["requires"] else "general"
+                print(f"  - {e['eval_id']}: {e['name']}")
+                print(f"    Requirements: {reqs}")
+            print()
+
+
+def main():
+    """CLI for exploring the eval registry."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Explore the eval registry")
+    parser.add_argument("--category", help="Filter by category")
+    parser.add_argument("--list", action="store_true", help="List all evals")
+    args = parser.parse_args()
+    
+    registry = EvalRegistry()
+    
+    if args.list:
+        registry.print_summary()
+    elif args.category:
+        evals = registry.get_by_category(args.category)
+        print(f"\nEvals in category '{args.category}':")
+        for e in evals:
+            print(f"  - {e['eval_id']}: {e['name']}")
+    else:
+        registry.print_summary()
+
+
+if __name__ == "__main__":
+    main()
+