research-copilot 0.1.0

package/lib/skills/data-analysis/SKILL.md

@@ -0,0 +1,285 @@
+---
+id: data-analysis-skill
+name: Data Analysis
+shortDescription: Python data analysis, visualization, statistics, and ML modeling
+category: Data & Analysis
+tools: [data-analyze]
+loadingStrategy: lazy
+tags: [data, analysis, python, visualization, statistics, ml]
+---
+
+Python data analysis guidance:
+- **Runtime Variables**: Use DATA_FILE, FIGURES_DIR, TABLES_DIR, DATA_DIR, RESULTS_FILE (pre-defined)
+- **STRICT OUTPUT RULE**: Generate ONLY what user requested (count nouns = count outputs)
+- **Results Manifest**: Always call write_results() at end with outputs and summary
+- **No Extras**: No bonus plots, summary tables, or supplementary files unless explicitly requested
+
+## Procedures
+
+### Critical Path Rules
+
+#### Pre-defined Runtime Variables
+The following variables are defined BEFORE your code runs:
+```python
+DATA_FILE    # Absolute path to input data file
+FIGURES_DIR  # Absolute path for saving figures
+TABLES_DIR   # Absolute path for saving CSV tables
+DATA_DIR     # Absolute path for transformed data
+RESULTS_FILE # Absolute path for results manifest JSON
+```
+
+#### Path Usage Rules (MUST FOLLOW)
+- Use `DATA_FILE` to read input: `pd.read_csv(DATA_FILE)`
+- Use `os.path.join(FIGURES_DIR, "name.png")` for outputs
+- Do NOT compute or derive paths
+- Do NOT use `os.path.dirname(__file__)`
+- Do NOT hardcode any file paths
+- Do NOT save to directories other than the pre-defined ones
+
+### STRICT MINIMAL OUTPUT RULE
+
+**This is critical—violation is a failure.**
+
+1. Count the nouns in user request:
+   - "a plot" = 1 figure
+   - "two charts" = 2 figures
+   - "statistics" = 1 summary table
+
+2. Generate EXACTLY that many outputs:
+   - User asks for "a plot" → produce 1 PNG, not 2, not 5
+   - User asks for "statistics" → produce 1 CSV, not 10
+
+3. Before every `plt.savefig()` or `df.to_csv()`:
+   - Ask: "Did the user request THIS specific output?"
+   - If NO → DELETE that code
+
+4. Do NOT create:
+   - Summary tables unless requested
+   - Intermediate DataFrames as CSV
+   - "Bonus" outputs like activity plots
+   - Supplementary analyses
+
+### Results Manifest
+
+Always call `write_results()` at the end:
+
+```python
+write_results(
+    outputs=[
+        {
+            "path": os.path.join(FIGURES_DIR, "scatter.png"),
+            "type": "figure",
+            "title": "X vs Y Scatter Plot",
+            "description": "Correlation visualization",  # optional
+            "tags": ["correlation", "scatter"]           # optional
+        },
+        {
+            "path": os.path.join(TABLES_DIR, "stats.csv"),
+            "type": "table",
+            "title": "Summary Statistics"
+        }
+    ],
+    summary={
+        "correlation": 0.85,
+        "n_rows": 1000,
+        "key_finding": "Strong positive correlation"
+    }
+)
+```
+
+### Analysis Tasks
+
+#### analyze (Statistical Analysis)
+- Compute only the statistics explicitly requested by the user
+- Identify correlations/outliers only when requested
+- Print key findings to stdout
+- Save summary as CSV table only if the user asked for file output
+
+#### visualize (Data Visualization)
+- Create appropriate plots for data types
+- Use matplotlib + seaborn for publication quality
+- Add titles, axis labels, legends
+- Use clean style (seaborn whitegrid)
+- Save as PNG (never plt.show())
+
+#### transform (Data Transformation)
+- Clean, reshape, or transform data
+- Handle missing values, type conversions
+- Save transformed dataset as CSV only when requested
+- Print summary of changes
+
+#### model (Statistical Modeling)
+- Build statistical/ML models (sklearn, statsmodels)
+- Report performance metrics
+- Save results summary as CSV only when requested
+- Print key metrics to stdout
+
+### Code Standards
+
+#### Required Setup
+```python
+import os
+import json
+import pandas as pd
+import numpy as np
+import matplotlib
+matplotlib.use('Agg')  # Non-interactive backend
+import matplotlib.pyplot as plt
+import seaborn as sns
+import warnings
+warnings.filterwarnings('ignore')
+```
+
+#### Figure Best Practices
+```python
+plt.figure(figsize=(10, 6), dpi=150)
+# ... plotting code ...
+plt.tight_layout()
+plt.savefig(os.path.join(FIGURES_DIR, "name.png"))
+plt.close()  # Always close after saving
+```
+
+#### Data Quality Checks
+```python
+# Check for missing values
+print(f"Missing values: {df.isnull().sum().sum()}")
+
+# Check data types
+print(df.dtypes)
+
+# Basic statistics
+print(df.describe())
+```
+
+## Examples
+
+### Single Plot Request
+
+User: "Create a scatter plot of column A vs column B"
+
+```python
+import os
+import pandas as pd
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+# Read data
+df = pd.read_csv(DATA_FILE)
+
+# Create the ONE requested plot
+plt.figure(figsize=(10, 6), dpi=150)
+sns.scatterplot(data=df, x='A', y='B')
+plt.title('A vs B')
+plt.xlabel('A')
+plt.ylabel('B')
+plt.tight_layout()
+plt.savefig(os.path.join(FIGURES_DIR, 'scatter_a_vs_b.png'))
+plt.close()
+
+# Write results manifest
+write_results(
+    outputs=[
+        {"path": os.path.join(FIGURES_DIR, 'scatter_a_vs_b.png'), "type": "figure", "title": "A vs B Scatter"}
+    ],
+    summary={"correlation": df['A'].corr(df['B'])}
+)
+```
+
+### Statistics Request
+
+User: "Calculate summary statistics for the dataset"
+
+```python
+import os
+import pandas as pd
+
+df = pd.read_csv(DATA_FILE)
+
+# Calculate statistics
+stats = df.describe()
+stats.to_csv(os.path.join(TABLES_DIR, 'summary_stats.csv'))
+
+# Print to stdout
+print("Summary Statistics:")
+print(stats)
+
+write_results(
+    outputs=[
+        {"path": os.path.join(TABLES_DIR, 'summary_stats.csv'), "type": "table", "title": "Summary Statistics"}
+    ],
+    summary={"n_rows": len(df), "n_cols": len(df.columns)}
+)
+```
+
+### Anti-Pattern: Over-Generation
+
+User: "Create a histogram of values"
+
+BAD (creates 5 outputs for 1 request):
+```python
+# Histogram
+plt.savefig('histogram.png')
+# Box plot (NOT REQUESTED)
+plt.savefig('boxplot.png')
+# Summary stats (NOT REQUESTED)
+stats.to_csv('stats.csv')
+# Data sample (NOT REQUESTED)
+df.head(100).to_csv('sample.csv')
+# Correlation matrix (NOT REQUESTED)
+plt.savefig('correlation.png')
+```
+
+GOOD (creates exactly 1 output):
+```python
+plt.figure(figsize=(10, 6), dpi=150)
+plt.hist(df['values'], bins=30, edgecolor='black')
+plt.title('Distribution of Values')
+plt.xlabel('Value')
+plt.ylabel('Frequency')
+plt.tight_layout()
+plt.savefig(os.path.join(FIGURES_DIR, 'histogram.png'))
+plt.close()
+
+write_results(
+    outputs=[{"path": os.path.join(FIGURES_DIR, 'histogram.png'), "type": "figure", "title": "Value Distribution"}]
+)
+```
+
+## Troubleshooting
+
+### "FileNotFoundError for data file"
+- Use DATA_FILE variable, not hardcoded path
+- Check: `print(DATA_FILE)` to verify path
+- Ensure file exists before reading
+
+### "Permission denied saving figure"
+- Use FIGURES_DIR, TABLES_DIR, DATA_DIR only
+- Do not save to arbitrary directories
+- Check directory exists: `os.makedirs(FIGURES_DIR, exist_ok=True)`
+
+### "Figure not appearing in results"
+- Call `plt.close()` after `plt.savefig()`
+- Use `matplotlib.use('Agg')` before importing pyplot
+- Include figure in write_results() outputs list
+
+### "Results manifest missing"
+- Always call write_results() at end of script
+- Check RESULTS_FILE path is used correctly
+- Ensure outputs list includes all generated files
+
+### "Too many outputs generated"
+- Re-read user request: count the nouns
+- Delete any code that creates unrequested outputs
+- Ask: "Did user ask for THIS?" for each savefig/to_csv
+
+### "Plot quality issues"
+- Set `dpi=150` for good resolution
+- Use `plt.tight_layout()` to prevent clipping
+- Add proper titles and labels
+
+### "Memory issues with large data"
+- Use chunked reading: `pd.read_csv(DATA_FILE, chunksize=10000)`
+- Select only needed columns
+- Use appropriate dtypes: `dtype={'col': 'category'}`

package/lib/skills/generate-skill-content.mjs

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+/**
+ * Build-time script: reads SKILL.md files and generates _generated.ts
+ * with inlined Markdown content as string constants.
+ *
+ * This eliminates all runtime filesystem reads, making skills work
+ * identically in unbundled ESM (tsx/node) and bundled (electron-vite).
+ *
+ * SKILL.md files remain the single source of truth.
+ *
+ * Usage:
+ *   node examples/research-pilot/skills/generate-skill-content.mjs
+ */
+
+import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs'
+import { join, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+
+// Auto-discover: any subdirectory containing SKILL.md (exclude default-project-skills)
+const skillDirs = readdirSync(__dirname).filter(name => {
+  if (name === 'default-project-skills') return false
+  if (name.startsWith('_') || name.startsWith('.')) return false
+  const fullPath = join(__dirname, name)
+  return statSync(fullPath).isDirectory() && (() => {
+    try { readFileSync(join(fullPath, 'SKILL.md'), 'utf-8'); return true } catch { return false }
+  })()
+})
+
+const exports = []
+const entries = []
+
+for (const dir of skillDirs.sort()) {
+  const content = readFileSync(join(__dirname, dir, 'SKILL.md'), 'utf-8')
+  const varName = dir.replace(/-([a-z])/g, (_, c) => c.toUpperCase()) + 'Content'
+  exports.push(`export const ${varName} = ${JSON.stringify(content)}`)
+  entries.push(`  '${dir}': ${varName},`)
+}
+
+const output = `/**
+ * AUTO-GENERATED — do not edit manually.
+ * Run: node examples/research-pilot/skills/generate-skill-content.mjs
+ *
+ * Source of truth: SKILL.md files in sibling directories.
+ */
+
+${exports.join('\n\n')}
+
+/** All skill content keyed by directory name */
+export const skillContent: Record<string, string> = {
+${entries.join('\n')}
+}
+`
+
+const outPath = join(__dirname, '_generated.ts')
+writeFileSync(outPath, output, 'utf-8')
+console.log(`[generate-skill-content] wrote ${outPath} (${skillDirs.length} skills: ${skillDirs.join(', ')})`)

package/lib/skills/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Research Copilot Skills
+ *
+ * Re-exports the skill loader and provides metadata for UI display.
+ */
+
+import { loadAllSkills, loadBuiltinSkills, loadWorkspaceSkills, getSkillByName, buildSkillsCatalog } from './loader.js'
+import type { SkillEntry, SkillCatalogItem } from './loader.js'
+
+export { loadAllSkills, loadBuiltinSkills, loadWorkspaceSkills, getSkillByName, buildSkillsCatalog }
+export type { SkillEntry, SkillCatalogItem }
+
+/** Metadata for UI display */
+export const researchPilotSkills = [
+  { id: 'data-analysis-skill', name: 'Data Analysis' },
+  // builtin skills
+  { id: 'scientific-writing', name: 'Scientific Writing' },
+  { id: 'scientific-visualization', name: 'Scientific Visualization' },
+  { id: 'research-grants', name: 'Research Grants' },
+  { id: 'matplotlib', name: 'Matplotlib' },
+  { id: 'seaborn', name: 'Seaborn' },
+  { id: 'scholar-evaluation', name: 'Scholar Evaluation' },
+  { id: 'rewrite-humanize', name: 'Rewrite & Humanize' }
+]
+
+/** Skill lookup by ID */
+export const skillsById = Object.fromEntries(
+  researchPilotSkills.map((s) => [s.id, s])
+) as Record<string, (typeof researchPilotSkills)[number]>
+
+/** Get skill metadata by ID */
+export function getSkill(id: string) {
+  return skillsById[id]
+}