microevals 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. config/judge_system_prompt.yaml +113 -0
  2. evals/nextjs/001-server-component.yaml +28 -0
  3. evals/nextjs/002-client-component.yaml +26 -0
  4. evals/nextjs/003-cookies.yaml +28 -0
  5. evals/nextjs/010-route-handlers.yaml +30 -0
  6. evals/nextjs/013-pathname-server.yaml +29 -0
  7. evals/nextjs/014-server-routing.yaml +28 -0
  8. evals/nextjs/018-use-router.yaml +28 -0
  9. evals/nextjs/020_no_use_effect.yaml +30 -0
  10. evals/nextjs/021-avoid-fetch-in-effect.yaml +28 -0
  11. evals/nextjs/022_prefer_server_actions.yaml +29 -0
  12. evals/nextjs/023_avoid_getserversideprops.yaml +27 -0
  13. evals/nextjs/024_avoid_redundant_usestate.yaml +29 -0
  14. evals/nextjs/025_no_async_client_components.yaml +29 -0
  15. evals/nextjs/026_no_serial_await.yaml +26 -0
  16. evals/nextjs/027-prefer-next-image.yaml +30 -0
  17. evals/nextjs/027_no_hooks_in_server_components.yaml +29 -0
  18. evals/nextjs/028-prefer-next-font.yaml +30 -0
  19. evals/nextjs/028_cookies_headers_context.yaml +29 -0
  20. evals/nextjs/029_no_catch_redirect.yaml +31 -0
  21. evals/nextjs/030_app_router_migration.yaml +30 -0
  22. evals/nextjs/031_no_non_serializable_props.yaml +31 -0
  23. evals/react/001_missing_useeffect_dependencies.yaml +29 -0
  24. evals/react/002_incorrect_event_handler.yaml +28 -0
  25. evals/react/003_missing_return_in_map.yaml +28 -0
  26. evals/react/004_async_useeffect.yaml +32 -0
  27. evals/react/005_direct_state_mutation.yaml +30 -0
  28. evals/react/006_index_as_key.yaml +31 -0
  29. evals/react/zustand_store_usage.yaml +25 -0
  30. evals/shadcn/001_cn_utility_function.yaml +31 -0
  31. evals/shadcn/002_css_variables.yaml +32 -0
  32. evals/shadcn/003_component_dependencies.yaml +33 -0
  33. evals/shadcn/004_path_aliases.yaml +32 -0
  34. evals/shadcn/005_client_directive.yaml +31 -0
  35. evals/shadcn/006_tailwind_config.yaml +36 -0
  36. evals/shadcn/007_components_json_config.yaml +35 -0
  37. evals/supabase/001_client_setup.yaml +47 -0
  38. evals/supabase/002_auth_context_setup.yaml +43 -0
  39. evals/supabase/003_auth_flow_implementation.yaml +46 -0
  40. evals/supabase/004_auth_flow_testing_WIP.yaml +52 -0
  41. evals/supabase/005_auth_google_oauth.yaml +55 -0
  42. evals/supabase/007_storage_client_setup.yaml +43 -0
  43. evals/supabase/008_storage_nextjs_config.yaml +45 -0
  44. evals/supabase/009_storage_image_upload.yaml +49 -0
  45. evals/supabase/010_security_rls_enabled.yaml +42 -0
  46. evals/supabase/011_security_rls_policies.yaml +43 -0
  47. evals/supabase/012_security_no_service_key_exposed.yaml +49 -0
  48. evals/supabase/013_database_read_data.yaml +44 -0
  49. evals/supabase/014_database_create_data.yaml +44 -0
  50. evals/supabase/015_database_update_data.yaml +47 -0
  51. evals/supabase/016_database_delete_data.yaml +47 -0
  52. evals/supabase/017_database_user_scoped_query.yaml +52 -0
  53. evals/tailwind/001_tailwind_v4_config.yaml +22 -0
  54. evals/tailwind/002_content_paths.yaml +27 -0
  55. evals/tailwind/003_no_dynamic_class_construction.yaml +28 -0
  56. evals/tailwind/tailwind_postcss_config.yaml +24 -0
  57. evals/typescript/001_unsafe_type_assertions.yaml +39 -0
  58. evals/typescript/002_missing_null_checks.yaml +33 -0
  59. evals/vercel/001_vercel_deployment.yaml +19 -0
  60. evals/vercel/002_environment_variables_handling.yaml +23 -0
  61. evals/vercel/003_seo_metadata.yaml +33 -0
  62. microevals/__init__.py +34 -0
  63. microevals/eval_registry.py +222 -0
  64. microevals/eval_runner.py +533 -0
  65. microevals/utils.py +490 -0
  66. microevals-0.1.0.dist-info/METADATA +575 -0
  67. microevals-0.1.0.dist-info/RECORD +71 -0
  68. microevals-0.1.0.dist-info/WHEEL +5 -0
  69. microevals-0.1.0.dist-info/entry_points.txt +2 -0
  70. microevals-0.1.0.dist-info/licenses/LICENSE +21 -0
  71. microevals-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,31 @@
1
+ eval_id: nextjs_no_catch_redirect_029
2
+ name: "No redirect()/notFound() in Try-Catch"
3
+ description: "Catches redirect/notFound in try-catch which breaks navigation"
4
+ category: nextjs
5
+
6
+ criteria: |
7
+ You have access to the entire codebase. Look for a subtle but critical bug.
8
+
9
+ CONTEXT: redirect() and notFound() work by throwing special errors that Next.js catches.
10
+
11
+ WHAT TO LOOK FOR:
12
+ Find any try-catch blocks that contain redirect() or notFound() calls.
13
+
14
+ ANTI-PATTERN (mark as failed - THIS IS A BUG):
15
+ - try { redirect('/login') } catch (e) { ... }
16
+ - try { if (condition) notFound() } catch (error) { ... }
17
+ - Catching the error prevents Next.js from handling navigation
18
+ - Results in broken redirects or 404 pages not showing
19
+
20
+ CORRECT PATTERN (mark as passed):
21
+ - redirect() and notFound() are NOT inside try-catch blocks
22
+ - If you need validation, do it BEFORE calling redirect/notFound
23
+ - Let these functions throw - Next.js will catch them correctly
24
+
25
+ SCORING:
26
+ - Score 1.0 (PASS): redirect() and notFound() are NOT inside try-catch blocks, no anti-patterns found
27
+ - Score 0.0 (FAIL): Anti-pattern found - redirect() or notFound() called inside try-catch (THIS IS A BUG that breaks navigation)
28
+ - Score -1.0 (N/A): No usage of redirect() or notFound() OR no Next.js App Router
29
+
30
+ NOTE: This is a subtle bug that makes navigation silently fail.
31
+
@@ -0,0 +1,30 @@
1
+ eval_id: nextjs_app_router_migration_030
2
+ name: "App Router Migration (No Pages Router Patterns)"
3
+ description: "Checks for Pages Router patterns in App Router apps"
4
+ category: nextjs
5
+
6
+ criteria: |
7
+ Check for Pages Router patterns in App Router apps.
8
+
9
+ OLD PATTERNS (Pages Router):
10
+ - pages/ directory with route files
11
+ - _app.js, _document.js special files
12
+ - getServerSideProps, getStaticProps, getInitialProps
13
+ - import from 'next/router' (should use 'next/navigation')
14
+
15
+ WHY IT'S WRONG:
16
+ - Mixing routers causes confusion and bugs
17
+ - App Router patterns don't work in Pages Router files
18
+ - Different mental models for routing and data fetching
19
+ - Can't leverage new App Router features
20
+
21
+ CORRECT APPROACH (App Router):
22
+ - app/ directory with route folders
23
+ - layout.js, page.js, loading.js, error.js
24
+ - Server components with async/await
25
+ - import from 'next/navigation' (useRouter, usePathname, etc.)
26
+
27
+ SCORING:
28
+ - Score 1.0 (PASS): Fully migrated to App Router - no Pages Router patterns
29
+ - Score 0.0 (FAIL): App Router exists but old Pages Router patterns still present
30
+ - Score -1.0 (N/A): No app/ directory exists (pure Pages Router app, which is fine for legacy apps)
@@ -0,0 +1,31 @@
1
+ eval_id: nextjs_no_non_serializable_props_031
2
+ name: "No Non-Serializable Props to Server Components"
3
+ description: "Catches passing functions to server components - causes runtime error"
4
+ category: nextjs
5
+
6
+ criteria: |
7
+ You have access to the entire codebase. Look for non-serializable props passed to server components.
8
+
9
+ CONTEXT: Server components can only receive JSON-serializable props.
10
+
11
+ WHAT TO LOOK FOR:
12
+ Find server components (files without 'use client') being used with function props.
13
+
14
+ ANTI-PATTERN (mark as failed - THIS IS A BUG):
15
+ - Passing onClick, onChange, or other function props to a server component
16
+ - Example: <ServerComponent onClick={handler} /> where ServerComponent has no 'use client'
17
+ - Error: "Functions cannot be passed to Client Components"
18
+ - Also applies to class instances, Dates, symbols
19
+
20
+ CORRECT PATTERN (mark as passed):
21
+ - Server components only receive serializable props (strings, numbers, plain objects, arrays)
22
+ - Components that need event handlers have 'use client' directive
23
+ - Or wrap the interactive part in a client component
24
+
25
+ SCORING:
26
+ - Score 1.0 (PASS): Server components only receive serializable props (strings, numbers, objects, arrays), no anti-patterns found
27
+ - Score 0.0 (FAIL): Anti-pattern found - passing functions or non-serializable data to server components (THIS IS A BUG)
28
+ - Score -1.0 (N/A): No custom components OR all components are client components OR no Next.js App Router
29
+
30
+ NOTE: Look at the component definition - if it has no 'use client' and receives function props, that's the bug.
31
+
@@ -0,0 +1,29 @@
1
+ eval_id: react_missing_useeffect_dependencies_001
2
+ name: "Missing Dependencies in useEffect"
3
+ description: "Checks if useEffect hooks include all dependencies in their dependency array"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect useEffect with missing dependencies causing stale closures.
8
+
9
+ ANTI-PATTERN:
10
+ - useEffect(() => { doSomething(prop) }, []) // prop not in deps
11
+ - useEffect(() => { fetch(url) }, []) // url not in deps
12
+ - Using props/state inside effect without listing them
13
+
14
+ WHY IT'S WRONG:
15
+ - Stale closures - effect uses old values
16
+ - Effect doesn't re-run when dependencies change
17
+ - Subtle bugs that are hard to debug
18
+ - React ESLint rule: exhaustive-deps
19
+
20
+ CORRECT:
21
+ - useEffect(() => { doSomething(prop) }, [prop])
22
+ - Include ALL variables from outer scope used in effect
23
+ - Or use useCallback/useMemo to stabilize references
24
+
25
+ SCORING:
26
+ - Score 1.0 (PASS): useEffect hooks exist AND all include correct dependencies (no stale closures)
27
+ - Score 0.0 (FAIL): useEffect hooks exist BUT missing dependencies found (list specific locations)
28
+ - Score -1.0 (N/A): No useEffect hooks found in codebase
29
+
@@ -0,0 +1,28 @@
1
+ eval_id: react_incorrect_event_handler_002
2
+ name: "Incorrect Event Handler Syntax"
3
+ description: "Checks for event handlers that are called immediately instead of on event"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect event handlers with incorrect syntax causing immediate execution.
8
+
9
+ ANTI-PATTERN:
10
+ - onClick={handleClick()} // calls immediately, not on click
11
+ - onChange={setValue(e.target.value)} // calls during render
12
+ - onSubmit={submitForm(data)} // calls immediately
13
+
14
+ WHY IT'S WRONG:
15
+ - Function executes during render, not on event
16
+ - Infinite render loops if it updates state
17
+ - Common LLM mistake
18
+
19
+ CORRECT:
20
+ - onClick={handleClick} // pass reference
21
+ - onClick={() => handleClick()} // arrow function wrapper
22
+ - onChange={(e) => setValue(e.target.value)} // arrow function
23
+
24
+ SCORING:
25
+ - Score 1.0 (PASS): Event handlers exist AND all use correct syntax (no immediate execution)
26
+ - Score 0.0 (FAIL): Event handlers exist BUT incorrect syntax found (list specific locations)
27
+ - Score -1.0 (N/A): No event handlers found in codebase
28
+
@@ -0,0 +1,28 @@
1
+ eval_id: react_missing_return_in_map_003
2
+ name: "Missing Return in Array.map()"
3
+ description: "Checks for missing return statements in array.map() causing undefined renders"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect array.map() with curly braces but missing return statement.
8
+
9
+ ANTI-PATTERN:
10
+ - array.map(item => { <Component /> }) // missing return
11
+ - items.map((item) => { <div>{item}</div> }) // missing return
12
+ - data.map(x => { const y = transform(x); <Element y={y} /> })
13
+
14
+ WHY IT'S WRONG:
15
+ - Returns undefined instead of component
16
+ - Nothing renders but no error shown
17
+ - Very common LLM mistake
18
+
19
+ CORRECT:
20
+ - array.map(item => <Component />) // implicit return
21
+ - array.map(item => { return <Component /> }) // explicit return
22
+ - array.map((item) => { const y = transform(item); return <Element y={y} /> })
23
+
24
+ SCORING:
25
+ - Score 1.0 (PASS): Array.map() with JSX exists AND all have proper returns (explicit or implicit)
26
+ - Score 0.0 (FAIL): Array.map() with JSX exists BUT missing return statements found (list specific locations)
27
+ - Score -1.0 (N/A): No array.map() with JSX found in codebase
28
+
@@ -0,0 +1,32 @@
1
+ eval_id: react_async_useeffect_004
2
+ name: "Async Function in useEffect Without Wrapper"
3
+ description: "Checks for async functions directly in useEffect (not allowed)"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect async functions used directly as useEffect callback.
8
+
9
+ ANTI-PATTERN:
10
+ - useEffect(async () => { await fetch() }, [])
11
+ - Using async directly as useEffect callback
12
+
13
+ WHY IT'S WRONG:
14
+ - useEffect expects function that returns void or cleanup function
15
+ - async functions always return Promise
16
+ - React warning: "Effect callbacks are synchronous"
17
+ - Can cause bugs with cleanup
18
+
19
+ CORRECT:
20
+ - useEffect(() => {
21
+ async function fetchData() { await fetch() }
22
+ fetchData()
23
+ }, [])
24
+ - Or: useEffect(() => {
25
+ (async () => { await fetch() })()
26
+ }, [])
27
+
28
+ SCORING:
29
+ - Score 1.0 (PASS): useEffect hooks exist AND none use async directly as callback (all properly wrapped)
30
+ - Score 0.0 (FAIL): useEffect hooks exist BUT async used directly as callback (list specific locations)
31
+ - Score -1.0 (N/A): No useEffect hooks found in codebase
32
+
@@ -0,0 +1,30 @@
1
+ eval_id: react_direct_state_mutation_005
2
+ name: "No Direct State Mutation"
3
+ description: "Checks for direct mutation of state instead of using setState"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect direct mutation of React state (arrays, objects).
8
+
9
+ ANTI-PATTERN:
10
+ - const [items, setItems] = useState([]); items.push(newItem)
11
+ - const [user, setUser] = useState({}); user.name = 'John'
12
+ - state.array[0] = newValue
13
+ - Mutating state directly without setState
14
+
15
+ WHY IT'S WRONG:
16
+ - React doesn't detect the change
17
+ - Component doesn't re-render
18
+ - Breaks React's state management
19
+ - Violates immutability principle
20
+
21
+ CORRECT:
22
+ - setItems([...items, newItem]) // spread operator
23
+ - setItems(prev => [...prev, newItem]) // functional update
24
+ - setUser({...user, name: 'John'}) // spread for objects
25
+
26
+ SCORING:
27
+ - Score 1.0 (PASS): State variables exist AND none are mutated directly (all use immutable updates)
28
+ - Score 0.0 (FAIL): State variables exist BUT direct mutations found (list specific locations)
29
+ - Score -1.0 (N/A): No state variables found in codebase
30
+
@@ -0,0 +1,31 @@
1
+ eval_id: react_index_as_key_006
2
+ name: "No Index as Key in Lists"
3
+ description: "Checks for using array index as key prop (causes bugs with reordering)"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect using array index as key prop in lists.
8
+
9
+ ANTI-PATTERN:
10
+ - items.map((item, index) => <div key={index}>{item}</div>)
11
+ - Using index as key prop
12
+ - Using i, idx, or other index variables as key
13
+
14
+ WHY IT'S WRONG:
15
+ - Keys should be stable and unique identifiers
16
+ - Index changes when items are reordered/filtered/sorted
17
+ - Causes state to be associated with wrong items
18
+ - Performance issues and rendering bugs
19
+ - Common LLM mistake
20
+
21
+ CORRECT:
22
+ - items.map(item => <div key={item.id}>{item.name}</div>)
23
+ - Use unique, stable ID from data
24
+ - If no ID, use combination of unique properties
25
+ - Only use index if list never changes order
26
+
27
+ SCORING:
28
+ - Score 1.0 (PASS): Lists with key props exist AND all use stable IDs (no index as key)
29
+ - Score 0.0 (FAIL): Lists with key props exist BUT index used as key (list specific locations)
30
+ - Score -1.0 (N/A): No lists with key props found in codebase
31
+
@@ -0,0 +1,25 @@
1
+ eval_id: zustand_store_usage
2
+ name: "Zustand Store Hook Usage"
3
+ description: "Verify the agent correctly identifies and fixes improper Zustand store hook usage patterns"
4
+ category: react
5
+
6
+ criteria: |
7
+ Detect improper Zustand store hook usage patterns.
8
+
9
+ ANTI-PATTERNS:
10
+ - useMemo(() => useMyStore, []) - wrapping store hook
11
+ - useMemo(() => useMyStore(), []) - calling inside useMemo
12
+ - useMemo(() => create(...), []) - creating store in component
13
+
14
+ CORRECT:
15
+ - const store = useMyStore (direct assignment)
16
+ - const useMyStore = create<T>(...) (at module level)
17
+
18
+ SCORING:
19
+ - Score 1.0 (PASS): Zustand store usage exists AND used correctly (direct assignment, no wrapping)
20
+ - Score 0.0 (FAIL): Zustand store usage exists BUT wrapped incorrectly (list specific locations)
21
+ - Score -1.0 (N/A): No Zustand store usage found in codebase
22
+
23
+ inputs:
24
+ store_name: "auto-detect"
25
+
@@ -0,0 +1,31 @@
1
+ eval_id: shadcn_cn_utility_function_001
2
+ name: "CN Utility Function Required"
3
+ description: "Checks if the cn() utility function exists when shadcn components are used"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify cn() utility function exists for shadcn components.
8
+
9
+ ANTI-PATTERN:
10
+ - Using shadcn components without lib/utils.ts
11
+ - Missing cn() function that merges Tailwind classes
12
+ - Components imported but utilities not set up
13
+
14
+ WHY IT'S WRONG:
15
+ - All shadcn components use cn() function
16
+ - Runtime error: "cn is not defined"
17
+ - App crashes immediately
18
+
19
+ CORRECT:
20
+ - lib/utils.ts with: export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) }
21
+ - Requires: clsx and tailwind-merge packages
22
+ - Must match path alias in components.json
23
+
24
+ EVALUATION:
25
+ - Check if shadcn components exist (check for @/components/ui/)
26
+ - Verify lib/utils.ts exists with cn function
27
+ - Verify clsx and tailwind-merge in package.json
28
+ - If components exist but cn missing: score 0.0
29
+ - If no shadcn components: score -1.0 (N/A)
30
+
31
+
@@ -0,0 +1,32 @@
1
+ eval_id: shadcn_css_variables_002
2
+ name: "Required CSS Variables Defined"
3
+ description: "Checks if shadcn theme CSS variables are defined in globals.css"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify CSS variables for shadcn theme are defined.
8
+
9
+ ANTI-PATTERN:
10
+ - Using shadcn components without CSS variable definitions
11
+ - Missing :root and .dark selectors in globals.css
12
+ - Components render but colors are broken/missing
13
+
14
+ WHY IT'S WRONG:
15
+ - Components rely on CSS variables like --background, --foreground, etc.
16
+ - Without them, components have no colors or wrong colors
17
+ - Silent failure - no error, just broken UI
18
+
19
+ CORRECT:
20
+ - globals.css must have:
21
+ :root { --background, --foreground, --primary, --secondary, etc. }
22
+ - .dark { ... } for dark mode variables
23
+ - All required shadcn CSS variables defined
24
+
25
+ EVALUATION:
26
+ - Check if shadcn components exist
27
+ - Verify globals.css has :root with CSS variables
28
+ - Check for variables: --background, --foreground, --primary, --card, --border
29
+ - If components exist but variables missing: score 0.0
30
+ - If no shadcn components: score -1.0 (N/A)
31
+
32
+
@@ -0,0 +1,33 @@
1
+ eval_id: shadcn_component_dependencies_003
2
+ name: "Required Component Peer Dependencies"
3
+ description: "Checks if peer dependencies are installed for shadcn components that require them"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify peer dependencies for shadcn components that require them.
8
+
9
+ ANTI-PATTERN:
10
+ - Using Form component without react-hook-form and zod
11
+ - Using Calendar without date-fns or day.js
12
+ - Using Sonner toast without sonner package
13
+ - Using Command without cmdk
14
+
15
+ WHY IT'S WRONG:
16
+ - Runtime error: "Cannot find module 'react-hook-form'"
17
+ - Components crash when dependencies missing
18
+ - Each specialized component has specific requirements
19
+
20
+ CORRECT:
21
+ - Form needs: react-hook-form, zod, @hookform/resolvers
22
+ - Calendar needs: react-day-picker, date-fns
23
+ - Command needs: cmdk
24
+ - Sonner needs: sonner
25
+ - Check shadcn docs for each component's requirements
26
+
27
+ EVALUATION:
28
+ - Detect which shadcn components are used
29
+ - Check if required dependencies are in package.json
30
+ - If component used without dependency: score 0.0
31
+ - If no components with dependencies: score -1.0 (N/A)
32
+
33
+
@@ -0,0 +1,32 @@
1
+ eval_id: shadcn_path_aliases_004
2
+ name: "Correct Path Aliases Configuration"
3
+ description: "Checks if path aliases (@/components) are properly configured for shadcn imports"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify path aliases are configured correctly for shadcn component imports.
8
+
9
+ ANTI-PATTERN:
10
+ - Components import from '@/components/ui/button'
11
+ - But tsconfig.json has no @ alias configured
12
+ - Or alias points to wrong directory
13
+
14
+ WHY IT'S WRONG:
15
+ - Module not found error: "Cannot find module '@/components/ui/button'"
16
+ - Imports fail at build time
17
+ - Shadcn CLI generates imports with @ alias by default
18
+
19
+ CORRECT:
20
+ - tsconfig.json (or jsconfig.json) must have:
21
+ "paths": { "@/*": ["./*"] } or "@/*": ["./src/*"]
22
+ - Must match components.json aliases configuration
23
+ - baseUrl must be set (usually ".")
24
+
25
+ EVALUATION:
26
+ - Check if shadcn components exist with @ imports
27
+ - Verify tsconfig.json/jsconfig.json has @ alias configured
28
+ - Check if path matches actual component location
29
+ - If imports use @ but not configured: score 0.0
30
+ - If no shadcn components: score -1.0 (N/A)
31
+
32
+
@@ -0,0 +1,31 @@
1
+ eval_id: shadcn_client_directive_005
2
+ name: "Use Client Directive for Interactive Components"
3
+ description: "Checks if interactive shadcn components have 'use client' directive in Next.js App Router"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify interactive shadcn components have 'use client' directive.
8
+
9
+ ANTI-PATTERN:
10
+ - Using Form, Dialog, Popover, etc. in server components
11
+ - Missing 'use client' in component files
12
+ - Importing client components directly in server components
13
+
14
+ WHY IT'S WRONG:
15
+ - Error: "You're importing a component that needs useState/useEffect"
16
+ - Interactive components require client-side rendering
17
+ - Hydration errors in Next.js App Router
18
+
19
+ CORRECT:
20
+ - Interactive components need 'use client' at top of file
21
+ - Or wrap in a client component wrapper
22
+ - Components that need it: Form, Dialog, Popover, DropdownMenu, Sheet, etc.
23
+
24
+ EVALUATION:
25
+ - Check if using Next.js App Router (app directory)
26
+ - Detect shadcn interactive components (form, dialog, dropdown, etc.)
27
+ - Verify these components or their wrappers have 'use client'
28
+ - If interactive components without 'use client': score 0.0
29
+ - If no App Router or no interactive components: score -1.0 (N/A)
30
+
31
+
@@ -0,0 +1,36 @@
1
+ eval_id: shadcn_tailwind_config_006
2
+ name: "Tailwind Config Theme Extension"
3
+ description: "Checks if Tailwind config includes required theme extension for shadcn CSS variables"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify Tailwind config has theme extension for shadcn CSS variables.
8
+
9
+ ANTI-PATTERN:
10
+ - Using shadcn without extending theme with CSS variables
11
+ - Missing color definitions in tailwind.config
12
+ - Components don't inherit theme colors properly
13
+
14
+ WHY IT'S WRONG:
15
+ - Tailwind doesn't recognize CSS variable-based colors
16
+ - Utilities like bg-background, text-foreground don't work
17
+ - Components may have inline styles but no utility support
18
+
19
+ CORRECT:
20
+ - tailwind.config must extend theme.colors:
21
+ colors: {
22
+ background: "hsl(var(--background))",
23
+ foreground: "hsl(var(--foreground))",
24
+ primary: { DEFAULT: "hsl(var(--primary))", foreground: "hsl(var(--primary-foreground))" },
25
+ // etc.
26
+ }
27
+ - borderRadius with CSS variables
28
+
29
+ EVALUATION:
30
+ - Check if shadcn components exist
31
+ - Verify tailwind.config extends theme with CSS variable colors
32
+ - Look for hsl(var(--background)) pattern in colors
33
+ - If components exist but theme not extended: score 0.0
34
+ - If no shadcn components: score -1.0 (N/A)
35
+
36
+
@@ -0,0 +1,35 @@
1
+ eval_id: shadcn_components_json_config_007
2
+ name: "Valid components.json Configuration"
3
+ description: "Checks if components.json exists and is properly configured for shadcn CLI"
4
+ category: shadcn
5
+
6
+ criteria: |
7
+ Verify components.json is properly configured for shadcn CLI.
8
+
9
+ ANTI-PATTERN:
10
+ - Missing components.json file
11
+ - Incorrect paths that don't match project structure
12
+ - Wrong alias configuration
13
+ - Style/baseColor mismatch with actual setup
14
+
15
+ WHY IT'S WRONG:
16
+ - shadcn CLI fails: "components.json not found"
17
+ - Components installed to wrong directory
18
+ - Import paths don't match actual paths
19
+ - Can't add new components with CLI
20
+
21
+ CORRECT:
22
+ - components.json at project root
23
+ - Correct aliases: { "components": "@/components", "utils": "@/lib/utils" }
24
+ - Valid paths matching tsconfig
25
+ - Style should match (default/new-york)
26
+ - tailwind config paths correct
27
+
28
+ EVALUATION:
29
+ - Check if shadcn components exist
30
+ - Verify components.json exists and is valid JSON
31
+ - Check if paths/aliases match actual project structure
32
+ - If components exist but config wrong: score 0.0
33
+ - If no shadcn components: score -1.0 (N/A)
34
+
35
+
@@ -0,0 +1,47 @@
1
+
2
+ eval_id: supabase_client_setup
3
+ name: "Supabase Client Setup and Configuration"
4
+ description: "Check if agent properly set up the Supabase client with correct configuration"
5
+ category: supabase
6
+
7
+ # What was the agent asked to do?
8
+ task_description: |
9
+ Build an application with Supabase backend
10
+
11
+ # How to evaluate success?
12
+ criteria: |
13
+ The agent was asked to build an application with Supabase backend.
14
+
15
+ DO NOT MODIFY ANY FILES. Only read and evaluate the existing code.
16
+
17
+ Evaluate if they completed the task successfully by checking:
18
+
19
+ 1. **Package Installation**
20
+ - Has @supabase/supabase-js package installed in package.json
21
+
22
+ 2. **Client Configuration**
23
+ - Creates Supabase client using createClient()
24
+ - Client is properly exported for use
25
+ - Uses environment variables (e.g., NEXT_PUBLIC_SUPABASE_URL, VITE_SUPABASE_URL, SUPABASE_URL, etc.)
26
+ - No hardcoded credentials in source code
27
+
28
+ 3. **Connection Validation**
29
+ - Test the connection using provided credentials: URL={supabase_url}, Key={supabase_anon_key}
30
+ - Run a basic test query to verify the connection works
31
+
32
+ Look at the repo and check for evidence like:
33
+ - package.json with @supabase/supabase-js dependency
34
+ - Supabase client file (lib/supabase.ts, utils/supabase.ts, config/supabase.ts, etc.)
35
+ - Proper use of environment variables
36
+ - Successful connection test
37
+
38
+ SCORING:
39
+ - 1.0 (PASS): All criteria met - package installed, client configured correctly, env vars used, connection works
40
+ - 0.0 (FAIL): Supabase was ATTEMPTED (package installed OR client imports OR env vars present) but incorrectly configured (hardcoded credentials, missing env vars, broken connection, etc.)
41
+ - -1.0 (N/A): Supabase is NOT USED AT ALL in the codebase - no @supabase/supabase-js dependency, no imports, no configuration attempts. This is the correct score when the project uses a different backend (Firebase, MongoDB, Razorpay, etc.)
42
+
43
+ # Optional: Custom inputs for this specific eval
44
+ inputs:
45
+ supabase_url: "SUPABASE_URL"
46
+ supabase_anon_key: "SUPABASE_ANON_KEY"
47
+ env_file_path: ".env.local"
@@ -0,0 +1,43 @@
1
+
2
+ eval_id: supabase_auth_context_setup
3
+ name: "Supabase Auth State Management Setup"
4
+ description: "Check if agent created auth state management with proper methods"
5
+ category: supabase
6
+
7
+ # What was the agent asked to do?
8
+ task_description: |
9
+ Create a login page and authentication system
10
+
11
+ # How to evaluate success?
12
+ criteria: |
13
+ The agent was asked to create a login page and authentication system.
14
+
15
+ DO NOT MODIFY ANY FILES. Only read and evaluate the existing code.
16
+
17
+ Evaluate if they completed the task successfully by checking:
18
+
19
+ 1. **Auth State Management Exists**
20
+ - Find auth context, store, service, or provider file
21
+ - Examples: contexts/auth.tsx, stores/auth.ts, services/auth.ts, composables/useAuth.ts, etc.
22
+ - Exports auth state management (provider, store, service, or composable)
23
+
24
+ 2. **Auth Methods Available**
25
+ - Exports auth interface/hook/composable (useAuth, authStore, authService, etc.)
26
+ - Provides user state
27
+ - Provides auth methods (signUp, signIn, signOut or similar)
28
+
29
+ Look at the repo and check for evidence like:
30
+ - Auth state management file (React Context, Vue store, Svelte store, Angular service, etc.)
31
+ - Exported auth interface (useAuth, authStore, authService, etc.)
32
+ - Auth methods available (signUp, signIn, signOut)
33
+
34
+ SCORING:
35
+ - 1.0 (PASS): Auth state management exists AND auth methods are available with user state
36
+ - 0.0 (FAIL): Missing auth state management OR missing auth methods OR anti-patterns found
37
+ - -1.0 (N/A): Supabase is NOT USED in the codebase (no @supabase/supabase-js dependency) OR no authentication system exists at all. If the app uses a different auth provider (Firebase Auth, Auth0, NextAuth, etc.), this is N/A.
38
+
39
+ # Optional: Custom inputs for this specific eval
40
+ inputs:
41
+ supabase_url: "SUPABASE_URL"
42
+ supabase_anon_key: "SUPABASE_ANON_KEY"
43
+