wtf-p 0.3.0 → 0.4.0

package/bin/lib/semantic-scholar.js

@@ -0,0 +1,184 @@
+const https = require('https');
+const querystring = require('querystring');
+
+/**
+ * Semantic Scholar API Wrapper
+ * 
+ * Provides search and lookup capabilities for Semantic Scholar Graph API v1.
+ * Handles rate limiting with exponential backoff.
+ * 
+ * Env:
+ *  - S2_API_KEY: Optional API key for higher rate limits.
+ */
+
+const API_KEY = process.env.S2_API_KEY;
+const BASE_HOST = 'api.semanticscholar.org';
+const BASE_PATH = '/graph/v1';
+
+const RETRY_CONFIG = {
+  maxRetries: 5,
+  baseDelay: 1000,
+  maxDelay: 16000
+};
+
+class SemanticScholarError extends Error {
+  constructor(message, statusCode, retryAfter) {
+    super(message);
+    this.name = 'SemanticScholarError';
+    this.statusCode = statusCode;
+    this.retryAfter = retryAfter;
+  }
+}
+
+/**
+ * Sleep helper
+ */
+const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Make an HTTP request with retry logic
+ */
+async function request(endpoint, params = {}) {
+  const query = querystring.stringify(params);
+  const path = `${BASE_PATH}${endpoint}?${query}`;
+  
+  let attempt = 0;
+
+  while (attempt <= RETRY_CONFIG.maxRetries) {
+    try {
+      return await doRequest(path);
+    } catch (error) {
+      if (error instanceof SemanticScholarError && error.statusCode === 429) {
+        attempt++;
+        if (attempt > RETRY_CONFIG.maxRetries) throw error;
+        
+        // Use Retry-After header if available, else exponential backoff
+        const delay = error.retryAfter 
+          ? parseInt(error.retryAfter, 10) * 1000 
+          : Math.min(RETRY_CONFIG.maxDelay, RETRY_CONFIG.baseDelay * Math.pow(2, attempt - 1));
+        
+        // console.error(`[S2] Rate limited. Retrying in ${delay}ms...`);
+        await sleep(delay);
+        continue;
+      }
+      throw error;
+    }
+  }
+}
+
+function doRequest(path) {
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: BASE_HOST,
+      path: path,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'WTF-P/0.4.0 (citation-expert)',
+      }
+    };
+
+    if (API_KEY) {
+      options.headers['x-api-key'] = API_KEY;
+    }
+
+    const req = https.request(options, (res) => {
+      let data = '';
+
+      res.on('data', (chunk) => data += chunk);
+
+      res.on('end', () => {
+        if (res.statusCode === 429) {
+          const retryAfter = res.headers['retry-after'];
+          return reject(new SemanticScholarError('Rate limit exceeded', 429, retryAfter));
+        }
+
+        if (res.statusCode >= 400) {
+          return reject(new SemanticScholarError(`API Error: ${res.statusCode} ${data}`, res.statusCode));
+        }
+
+        try {
+          const json = JSON.parse(data);
+          resolve(json);
+        } catch (e) {
+          reject(new Error(`Failed to parse response: ${e.message}`));
+        }
+      });
+    });
+
+    req.on('error', (e) => reject(new Error(`Request failed: ${e.message}`)));
+    req.on('timeout', () => {
+      req.destroy();
+      reject(new Error('Request timed out'));
+    });
+    
+    // Set a reasonable timeout (10s)
+    req.setTimeout(10000);
+    req.end();
+  });
+}
+
+/**
+ * Search for papers
+ * @param {string} query 
+ * @param {Object} options 
+ */
+async function search(query, options = {}) {
+  const limit = Math.min(options.limit || 10, 100);
+  const fields = options.fields || [
+    'paperId', 'externalIds', 'title', 'abstract', 'venue', 'year', 
+    'citationCount', 'influentialCitationCount', 'authors', 
+    'publicationTypes', 'openAccessPdf'
+  ];
+
+  const params = {
+    query,
+    limit,
+    fields: fields.join(',')
+  };
+
+  if (options.year) params.year = options.year;
+
+  const response = await request('/paper/search', params);
+  return response.data || [];
+}
+
+/**
+ * Get paper by ID
+ * @param {string} id - S2 PaperId, DOI, ArXivId, etc.
+ * @param {string[]} fields 
+ */
+async function getPaper(id, fields = []) {
+  const defaultFields = [
+    'paperId', 'externalIds', 'title', 'abstract', 'venue', 'year', 
+    'citationCount', 'influentialCitationCount', 'authors', 
+    'publicationTypes', 'openAccessPdf'
+  ];
+  
+  const params = {
+    fields: (fields.length > 0 ? fields : defaultFields).join(',')
+  };
+
+  return await request(`/paper/${id}`, params);
+}
+
+/**
+ * Get citations for a paper
+ * @param {string} paperId 
+ * @param {number} limit 
+ */
+async function getCitations(paperId, limit = 20) {
+  const params = {
+    limit,
+    fields: 'title,year,citationCount,authors' // Minimal fields for citations
+  };
+  
+  const response = await request(`/paper/${paperId}/citations`, params);
+  return response.data || [];
+}
+
+module.exports = {
+  search,
+  getPaper,
+  getCitations,
+  SemanticScholarError
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wtf-p",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "The G.O.A.T. meta-prompting system for 10x researchers who need to submit papers and projects and proposals YESTERDAY.",
   "bin": {
     "wtfp": "bin/install.js",

package/vendors/claude/agents/wtfp/citation-expert.md

@@ -0,0 +1,45 @@
+---
+name: citation-expert
+description: Academic citation specialist. Searches papers, manages BibTeX, and identifies literature gaps.
+allowed-tools:
+  - Bash
+  - Read
+  - Write
+---
+
+# Citation Expert
+
+You are a senior academic librarian and bibliometric specialist. Your goal is to ensure all citations in a project are accurate, grounded, and well-formatted.
+
+## Tools
+You have access to specialized CLI tools for citation management:
+
+1.  **Retrieval:** `node ~/.claude/bin/citation-fetcher.js "<query>"`
+    - Use this to find new papers and get their BibTeX.
+    - Source: CrossRef / Semantic Scholar.
+
+2.  **Bibliography Management:** `node ~/.claude/bin/bib-index.js <command> ...`
+    - `index <file>`: List all keys in a bib file.
+    - `get <file> <key>`: Read a specific entry.
+    - `search <file> <query>`: Find entries by keyword.
+
+## Capabilities
+
+1.  **Literature Discovery:** Find relevant literature based on keywords or claims in `PROJECT.md`.
+2.  **Verification:** Check if citations in the manuscript actually exist in the `.bib` file.
+3.  **Formatting:** Ensure BibTeX entries follow standard conventions.
+4.  **Gap Analysis:** Compare current citations against the core argument.
+
+## Principles
+
+- **Precision Over Recall:** Prefer a few highly relevant citations.
+- **No Direct Writes:** NEVER overwrite the user's `references.bib`. Always create a `suggested.bib` or provide the BibTeX in the chat.
+- **Grounding:** Always check the `PROJECT.md` context.
+- **Deterministic:** Use the provided tools to verify facts. Do not hallucinate citations.
+
+## Workflow
+
+1.  **Analyze:** Understand the user's need (search vs. fix).
+2.  **Execute:** Use the appropriate script.
+3.  **Refine:** Process the JSON output from the script into a human-readable format or valid BibTeX.
+4.  **Report:** Present findings or suggested changes.

package/vendors/claude/agents/wtfp/citation-formatter.md

@@ -0,0 +1,42 @@
+---
+name: citation-formatter
+description: Specialist in managing BibTeX files, checking citation consistency, and formatting references.
+allowed-tools:
+  - Bash
+  - Read
+  - Write
+---
+
+# Citation Formatter
+
+You are a bibliometric quality assurance specialist. Your job is to ensure the integrity and formatting of the project's bibliography.
+
+## Tools
+You have access to the Bibliography Indexer:
+- `node ~/.claude/bin/bib-index.js index <file>`: List all keys.
+- `node ~/.claude/bin/bib-index.js get <file> <key>`: Get full BibTeX for a key.
+- `node ~/.claude/bin/bib-index.js search <file> <query>`: Find entries.
+
+## Core Responsibilities
+
+1.  **Validation:** Check if citations in the text (e.g., `\cite{foo}`) actually exist in the `.bib` file.
+2.  **Formatting:** detailed formatting of BibTeX entries (e.g., ensuring curly braces around titles to preserve capitalization).
+3.  **Deduplication:** Identify potential duplicate entries.
+
+## Critical Rules
+
+1.  **Read-Only on Source:** You must **NEVER** overwrite the user's primary `.bib` file (usually `references.bib`).
+2.  **Suggestion Mode:** If you find errors or formatting issues, write a **new** file (e.g., `references_suggested.bib`) or output the corrected BibTeX in the chat.
+3.  **Deterministic:** Rely on the `bib-index.js` tool to verify existence. Do not guess.
+
+## Common Workflows
+
+- **"Check references":**
+  1. Read the latex/markdown files to find citations.
+  2. Index the `.bib` file.
+  3. Cross-reference and report missing keys.
+
+- **"Fix formatting":**
+  1. Read the entry using `get`.
+  2. Apply formatting rules (e.g., standardizing conference names).
+  3. Output the corrected block.

package/vendors/claude/agents/wtfp/citation-retriever.md

@@ -0,0 +1,31 @@
+---
+name: citation-retriever
+description: Specialist in finding academic literature and generating BibTeX.
+allowed-tools:
+  - Bash
+  - Read
+---
+
+# Citation Retriever
+
+You are an expert at discovering academic literature. Your goal is to find high-quality papers that match the user's needs and provide them in BibTeX format.
+
+## Tools
+You have access to a custom search tool via the command line:
+`node ~/.claude/bin/citation-fetcher.js "<query>"`
+
+This tool queries academic databases (like CrossRef) and returns a JSON list of papers, including a draft BibTeX entry.
+
+## Workflow
+1.  **Analyze Request:** Understand what the user is looking for.
+2.  **Contextualize:** If needed, read `.planning/PROJECT.md` to understand the broader research context.
+3.  **Search:** Run the fetcher script with specific keywords.
+4.  **Present:** Output the results. If the user asks for BibTeX, provide the entries generated by the tool.
+
+## Rules
+- **No Hallucinations:** Only provide citations returned by the tool.
+- **Verification:** If the tool returns no results, try broader keywords. Do not invent papers.
+- **Output:** When presenting BibTeX, use a code block:
+  ```bibtex
+  @article{...}
+  ```

package/vendors/claude/commands/wtfp/analyze-bib.md

@@ -43,60 +43,51 @@ BibTeX file: $ARGUMENTS (optional - auto-detects .bib files if not provided)
 find . -name "*.bib" -type f 2>/dev/null | grep -v node_modules | grep -v .git | head -10
 ```
 
-If multiple found, use AskUserQuestion:
-- header: "BibTeX"
-- question: "Which bibliography file should I analyze?"
-- options: [list found files] + "Analyze all"
+If multiple found, use AskUserQuestion.
+If none found, exit with error.
+</step>
 
-If none found:
-```
-No .bib file found.
+<step name="parse_entries">
+**Parse BibTeX Index:**
+
+Use the specialized indexer tool to get a structured JSON summary of the bibliography. This handles large files efficiently.
 
-Either:
-1. Add your .bib file to the project
-2. Specify path: /wtfp:analyze-bib path/to/refs.bib
+```bash
+# Index the bibliography (returns JSON)
+node ~/.claude/bin/bib-index.js index "$ARGUMENTS"
 ```
-Exit command.
+*(If $ARGUMENTS is empty, use the file found in previous step)*
 
 </step>
 
-<step name="parse_entries">
-**Parse all BibTeX entries:**
+<step name="impact_analysis">
+**Impact Analysis:**
 
-Extract from each entry:
-- Citation key
-- Type (@article, @inproceedings, @book, etc.)
-- Authors
-- Title
-- Year
-- Venue (journal/booktitle)
-- Abstract (if present)
-- Keywords (if present)
+Analyze citation metrics (citations, velocity, age) to identify seminal and rising papers.
 
-Build structured inventory of all references.
+```bash
+node ~/.claude/bin/analyze-impact.js "$ARGUMENTS"
+```
+
+Output sections:
+1. **Seminal Works** (>1000 citations)
+2. **Rising Stars** (High citation velocity)
+3. **Review Suggested** (Old, low impact)
 
 </step>
 
 <step name="temporal_analysis">
 **Temporal Analysis:**
 
-Categorize by publication year:
-- **Foundational** (10+ years old): Seminal works, established theory
-- **Established** (5-10 years): Mature approaches, proven methods
-- **Recent** (2-5 years): Current state of the art
-- **Cutting edge** (<2 years): Latest developments, concurrent work
+Analyze the JSON output from the parse_entries step.
 
-```markdown
-## Temporal Distribution
-
-| Era | Count | Examples |
-|-----|-------|----------|
-| Foundational (pre-2015) | [N] | [key1], [key2] |
-| Established (2015-2019) | [N] | [key3], [key4] |
-| Recent (2020-2023) | [N] | [key5], [key6] |
-| Cutting edge (2024+) | [N] | [key7], [key8] |
-```
+Categorize entries by the `year` field in the JSON:
+- **Foundational** (10+ years old)
+- **Established** (5-10 years)
+- **Recent** (2-5 years)
+- **Cutting edge** (<2 years)
 
+Produce the "Temporal Distribution" table based on these counts.
 </step>
 
 <step name="cluster_topics">
@@ -125,21 +116,16 @@ Analyze titles, abstracts, and keywords to cluster papers by theme:
 </step>
 
 <step name="identify_seminal">
-**Identify Seminal Works:**
+**Verify Seminal Works:**
 
-Flag likely seminal papers based on:
-- High citation count (if DOI available, could fetch)
-- Foundational publication year
-- Appears in prestigious venue
-- Authors are field leaders
-- Title suggests foundational contribution
+Review the "Seminal Works" list from the Impact Analysis step.
 
 ```markdown
-## Seminal Works (Must Cite)
+## Seminal Works (Verified High Impact)
 
-| Key | Title | Why Seminal |
-|-----|-------|-------------|
-| [key] | [title] | [reason: foundational method, influential framework, etc.] |
+| Key | Title | Citations | Why Seminal |
+|-----|-------|-----------|-------------|
+| [key] | [title] | [N] | [reason] |
 ```
 
 Use AskUserQuestion:
@@ -233,15 +219,18 @@ Write to `.planning/sources/REFS.md`:
 **Total entries:** [N]
 **Analysis date:** [date]
 
+## Impact Analysis
+[From impact_analysis step]
+- Seminal Works (>1000 citations)
+- Rising Stars (High Velocity)
+- Review Suggested (Low Impact/Old)
+
 ## Temporal Distribution
 [From temporal_analysis step]
 
 ## Topic Clusters
 [From cluster_topics step]
 
-## Seminal Works
-[From identify_seminal step]
-
 ## Citation Map by Section
 [From map_to_sections step]
 

package/vendors/claude/commands/wtfp/check-refs.md

@@ -55,16 +55,18 @@ Create one or specify path: /wtfp:check-refs path/to/refs.bib
 </step>
 
 <step name="parse_bib">
-**Extract all BibTeX keys:**
+**Extract BibTeX Keys:**
+
+Use the deterministic indexer to get an accurate list of keys from the .bib file.
 
 ```bash
-grep -E "^@[a-zA-Z]+\{" *.bib | sed 's/.*{\([^,]*\),.*/\1/'
+node ~/.claude/bin/bib-index.js index "$ARGUMENTS"
 ```
+*(Use the file found/selected in step 1)*
 
-Build inventory:
+Build inventory from the JSON output:
 - Entry key
-- Entry type (@article, @inproceedings, etc.)
-- Required fields present/missing
+- Title
 - Year
 </step>
 
@@ -133,10 +135,10 @@ Use AskUserQuestion:
   - "Keep all" — Leave .bib unchanged
   - "Comment out" — Keep but mark as unused
 
-For missing entries, offer to:
-- Search for DOI/metadata (if WebFetch available)
-- Create skeleton entry for user to complete
-- Flag for manual resolution
+For missing entries, offer to find them:
+1. **Search:** Run `node ~/.claude/bin/citation-fetcher.js "title or author"` (uses Semantic Scholar + CrossRef) to find the correct BibTeX.
+2. **Create:** If found, append the new entry to the .bib file.
+3. **Flag:** If not found, flag for manual resolution.
 </step>
 
 <step name="apply_fixes">

package/vendors/claude/commands/wtfp/new-paper.md

@@ -88,116 +88,77 @@ Exit command.
 
 </step>
 
-<step name="type">
+<step name="initial_batch">
 
-**Ask paper type:**
+**Gather Core Context (Batched):**
 
-Use AskUserQuestion:
-- header: "Paper Type"
-- question: "What type of document are you writing?"
+Use AskUserQuestion to collect the foundational pillars in one turn.
+
+- header: "Paper Foundations"
+- question: "To initialize the project, I need to understand 4 things. Answer what you can, and I'll infer or ask about the rest:\n\n1. **Type:** (Research paper, Grant, Thesis, etc.)\n2. **Venue:** (Target conference/journal, e.g., NeurIPS, CHI, Nature)\n3. **Core Argument:** (The one main thing you are proving/proposing)\n4. **Audience:** (Who is this for?)"
 - options:
-  - "Research paper" — Journal article or conference paper
-  - "Grant proposal" — Funding application (NSF, NIH, etc.)
-  - "Thesis chapter" — Dissertation or thesis section
-  - "Other" — Essay, report, or something else
+  - "I provided the details" — Continue with what I said
+  - "Guide me step-by-step" — I prefer individual questions
+
+**If "Guide me step-by-step":**
+Fall back to the sequential interview mode (ask Type, then Venue, then Argument).
 
-Store the document type for later structure decisions.
+**If "I provided the details":**
+Parse the response to extract: `document_type`, `target_venue`, `core_argument`, `target_audience`.
 
 </step>
 
 <step name="venue_template">
 
-**Select venue template (for research papers):**
+**Select venue template (if not inferred):**
 
-If document type is "Research paper", ask for venue template:
+If `document_type` or `target_venue` is ambiguous, ask specifically for the template:
 
 Use AskUserQuestion:
-- header: "Venue"
-- question: "What's your target venue or field?"
+- header: "Venue Structure"
+- question: "Based on your input, which structure fits best?"
 - options:
-  - "ACM CS" — Systems, databases, architecture (SIGMOD, OSDI, SOSP)
-  - "IEEE CS" — HPC, parallel systems, storage (SC, IPDPS, TPDS)
+  - "ACM CS" — Systems/Databases (SIGMOD, SOSP)
+  - "IEEE CS" — HPC/Systems (SC, IPDPS)
   - "ML/AI" — NeurIPS, ICML, ICLR style
-  - "Nature/Science" — Classic IMRaD for life sciences
-  - "Other" — Custom structure
+  - "Nature/Science" — Life Sciences IMRaD
+  - "Grant" — Funding Proposal
+  - "Thesis" — Dissertation Chapter
+  - "Other" — Custom
 
 **Load venue template:**
-
-Based on selection, read the corresponding venue template:
-- ACM CS → `~/.claude/write-the-f-paper/venues/acm-cs.yaml`
-- IEEE CS → `~/.claude/write-the-f-paper/venues/ieee-cs.yaml`
-- ML/AI → `~/.claude/write-the-f-paper/venues/arxiv-ml.yaml`
-- Nature/Science → `~/.claude/write-the-f-paper/venues/nature.yaml`
-- Other → Ask user to describe structure
-
-If document type is "Thesis chapter":
-- Load `~/.claude/write-the-f-paper/venues/thesis-chapter.yaml`
-
-Store the selected venue template for use in structure step.
+(Load corresponding YAML from `~/.claude/write-the-f-paper/venues/`)
 
 </step>
 
-<step name="question">
-
-**1. Open (FREEFORM — do NOT use AskUserQuestion):**
-
-Ask inline: "What is the core argument or contribution of your paper?"
-
-Wait for their freeform response. This gives you the context needed to ask intelligent follow-up questions.
-
-**2. Follow the thread (NOW use AskUserQuestion):**
+<step name="refine_argument">
 
-Based on their response, use AskUserQuestion with options that probe what they mentioned:
-- header: "[Topic they mentioned]"
-- question: "You mentioned [X] — how would you describe the key insight?"
-- options: 2-3 interpretations + "Something else"
+**Sharpen the Core:**
 
-**3. Target venue:**
+If `core_argument` was weak or missing in the batch, drill down now.
 
 Use AskUserQuestion:
-- header: "Venue"
-- question: "What's the target venue or format?"
-- options: Common venues for their field + "Undecided" + "Let me specify"
+- header: "Refine Core"
+- question: "Let's sharpen the core argument. If reviewers remember ONE thing, what is it?"
+- options:
+  - "The Method" — A new way of doing X
+  - "The Findings" — We discovered Y
+  - "The Theory" — A new perspective on Z
+  - "The System" — We built system Q
+  - "Write my own" — (User types input)
 
-**4. Sharpen the core:**
+</step>
 
-Use AskUserQuestion:
-- header: "Core"
-- question: "If reviewers remember one thing, what should it be?"
-- options: Key aspects they've mentioned + "The methodology" + "The findings" + "Something else"
+<step name="constraints_batch">
 
-**5. Find boundaries:**
+**Gather Constraints & Boundaries (Batched):**
 
 Use AskUserQuestion:
-- header: "Scope"
-- question: "What's explicitly NOT in this paper?"
-- options: Tempting tangents + "Nothing specific" + "Let me list them"
-
-**6. Constraints:**
-
-Use AskUserQuestion:
-- header: "Constraints"
-- question: "Any hard constraints?"
+- header: "Scope & Limits"
+- question: "Finally, define the box we are playing in:\n\n1. **Out of Scope:** What are we explicitly NOT doing?\n2. **Hard Constraints:** Page limits, deadlines, data restrictions?"
 - options:
-  - "Page/word limit" — Strict length requirements
-  - "Deadline" — Submission deadline pressure
-  - "Data limitations" — Working with specific dataset
-  - "None" — Flexible constraints
-  - "Multiple constraints" — Let me explain
-
-**7. Decision gate:**
-
-Use AskUserQuestion:
-- header: "Ready?"
-- question: "Ready to create PROJECT.md, or explore more?"
-- options (ALL THREE REQUIRED):
-  - "Create PROJECT.md" — Finalize and continue
-  - "Ask more questions" — I'll dig deeper
-  - "Let me add context" — You have more to share
-
-If "Ask more questions" → return to step 2.
-If "Let me add context" → receive input via their response → return to step 2.
-Loop until "Create PROJECT.md" selected.
+  - "Provided details" — Save and continue
+  - "No constraints" — Standard length, no specific exclusions
 
 </step>
 

package/vendors/claude/commands/wtfp/research-gap.md

@@ -73,10 +73,9 @@ Use AskUserQuestion:
 Based on scope, investigate:
 
 **Key Citations:**
-- Identify foundational papers
-- Find recent high-impact work
-- Note methodological papers
-- Track disagreements/debates
+- **Foundational:** Use `node ~/.claude/bin/citation-fetcher.js "[topic]" --intent=seminal` to find high-impact papers.
+- **Recent:** Use `node ~/.claude/bin/citation-fetcher.js "[topic]" --intent=recent --year=2023-2026` to find state-of-the-art.
+- **Validation:** Use `node ~/.claude/bin/citation-fetcher.js "query"` to verify specific papers.
 
 **Standard Approaches:**
 - How do others structure similar sections?