wtf-p 0.3.0 → 0.5.0

package/bin/lib/citation-fetcher.js

@@ -0,0 +1,299 @@
+const https = require('https');
+const querystring = require('querystring');
+const s2 = require('./semantic-scholar');
+const scholar = require('./scholar-lookup');
+const ranker = require('./citation-ranker');
+const bibFormat = require('./bib-format');
+
+/**
+ * WTF-P Citation Fetcher v0.4.0
+ * 
+ * Orchestrates tiered search:
+ * 1. Semantic Scholar (Primary, Free)
+ * 2. SerpAPI (Optional, Seminal/Paid)
+ * 3. CrossRef (Fallback)
+ * 
+ * Usage:
+ *   node citation-fetcher.js "<query>" --intent=<intent> --year=<year>
+ */
+
+// --- CrossRef Fallback ---
+async function searchCrossRef(query, limit = 5) {
+  return new Promise((resolve, reject) => {
+    const params = {
+      query: query,
+      rows: limit,
+      sort: 'relevance',
+      select: 'DOI,title,author,issued,type,container-title,volume,issue,page,abstract'
+    };
+
+    const options = {
+      hostname: 'api.crossref.org',
+      path: `/works?${querystring.stringify(params)}`,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'WTF-P/0.4.0 (citation-expert)'
+      }
+    };
+
+    const req = https.request(options, (res) => {
+      let data = '';
+      res.on('data', c => data += c);
+      res.on('end', () => {
+        if (res.statusCode !== 200) return resolve([]);
+        try {
+          const json = JSON.parse(data);
+          const items = json.message.items || [];
+          resolve(items.map(mapCrossRefToPaper));
+        } catch (e) {
+          resolve([]);
+        }
+      });
+    });
+    
+    req.on('error', () => resolve([]));
+    req.end();
+  });
+}
+
+function mapCrossRefToPaper(item) {
+  const title = item.title ? item.title[0] : 'Untitled';
+  const year = item.issued && item.issued['date-parts'] ? item.issued['date-parts'][0][0] : null;
+  
+  return {
+    source: 'crossref',
+    title: title,
+    year: year,
+    doi: item.DOI,
+    venue: item['container-title'] ? item['container-title'][0] : null,
+    authors: (item.author || []).map(a => ({ name: `${a.family}, ${a.given}` })),
+    abstract: item.abstract ? item.abstract.replace(/<[^>]*>?/gm, '').trim() : null,
+    citationCount: 0, // CrossRef doesn't give citation counts easily
+    externalIds: { DOI: item.DOI }
+  };
+}
+
+// --- Mapper ---
+
+function mapS2ToPaper(item) {
+  return {
+    source: 'semantic_scholar',
+    paperId: item.paperId,
+    title: item.title,
+    year: item.year,
+    venue: item.venue,
+    authors: item.authors || [],
+    citationCount: item.citationCount || 0,
+    abstract: item.abstract,
+    externalIds: item.externalIds || {},
+    doi: (item.externalIds && item.externalIds.DOI) || null,
+    openAccessPdf: item.openAccessPdf
+  };
+}
+
+function mapScholarToPaper(item) {
+  return {
+    source: 'google_scholar',
+    scholarClusterId: item.clusterId,
+    title: item.title,
+    year: item.year,
+    venue: item.venue,
+    authors: item.authors || [],
+    citationCount: item.citationCount || 0,
+    abstract: item.snippet, // Use snippet as abstract fallback
+    externalIds: {},
+    doi: null, // Scholar doesn't reliably give DOIs
+    openAccessPdf: item.link ? { url: item.link } : null
+  };
+}
+
+// --- Deduplication ---
+
+function fingerprint(title, year) {
+  if (!title) return `unknown::${Math.random()}`;
+  const normalized = title.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim();
+  return `${normalized}::${year || '????'}`;
+}
+
+function deduplicatePapers(papers) {
+  const seen = new Map();
+
+  for (const paper of papers) {
+    // Priority: DOI > S2ID > ScholarID > Fingerprint
+    const key = paper.doi 
+      || paper.paperId 
+      || paper.scholarClusterId
+      || fingerprint(paper.title, paper.year);
+
+    if (!seen.has(key)) {
+      seen.set(key, paper);
+    } else {
+      // Merge: prefer richer metadata
+      // S2 usually has best metadata (abstracts, verified authors).
+      // Scholar has best citation counts.
+      // CrossRef has verified DOIs.
+      
+      const existing = seen.get(key);
+      let merged = { ...existing };
+      
+      // If new one is S2, take its metadata but keep higher citation count
+      if (paper.source === 'semantic_scholar') {
+        merged = { ...paper, citationCount: Math.max(existing.citationCount, paper.citationCount) };
+        if (existing.scholarClusterId) merged.scholarClusterId = existing.scholarClusterId;
+      } 
+      // If new one is Scholar, just update citation count and maybe ID
+      else if (paper.source === 'google_scholar') {
+        merged.citationCount = Math.max(merged.citationCount, paper.citationCount);
+        merged.scholarClusterId = paper.clusterId;
+      }
+      
+      seen.set(key, merged);
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+// --- Main Search Logic ---
+
+async function search(query, options = {}) {
+  const limit = options.limit || 10;
+  const intent = options.intent || 'balanced';
+  
+  let papers = [];
+  const errors = [];
+  
+  const searchPromises = [];
+
+  // 1. S2 Search (Always)
+  searchPromises.push(
+    s2.search(query, { limit: limit * 2, year: options.year })
+      .then(res => res.map(mapS2ToPaper))
+      .catch(e => {
+        errors.push(`S2 Error: ${e.message}`);
+        return [];
+      })
+  );
+  
+  // 2. Scholar Search (Conditional)
+  if ((intent === 'seminal' || options.useScholar) && scholar.isAvailable()) {
+    searchPromises.push(
+      scholar.search(query, { limit: limit, yearLow: options.year, yearHigh: options.year })
+        .then(res => res.map(mapScholarToPaper))
+        .catch(e => {
+          errors.push(`Scholar Error: ${e.message}`);
+          return [];
+        })
+    );
+  }
+
+  const results = await Promise.all(searchPromises);
+  results.forEach(r => papers.push(...r));
+
+  // 3. CrossRef Fallback (if few results)
+  if (papers.length < 5) {
+    try {
+      const crResults = await searchCrossRef(query, limit);
+      papers.push(...crResults);
+    } catch (e) {
+      errors.push(`CrossRef Error: ${e.message}`);
+    }
+  }
+
+  // 4. Deduplicate
+  const unique = deduplicatePapers(papers);
+
+  // 5. Rank
+  const ranked = ranker.rank(unique, intent);
+
+  // 6. Format to BibTeX
+  const formatted = ranked.slice(0, limit).map(p => {
+    // Generate key
+    const firstAuthor = p.authors && p.authors.length > 0 
+      ? (p.authors[0].name ? p.authors[0].name.split(',')[0].trim().split(' ').pop().toLowerCase() : 'unknown') 
+      : 'unknown';
+      
+    // Handle S2 author format variants or string parsing
+    let familyName = 'unknown';
+    if (p.authors && p.authors.length > 0) {
+       const nameParts = p.authors[0].name.split(' ');
+       familyName = nameParts[nameParts.length - 1].toLowerCase().replace(/[^a-z]/g, '');
+    }
+
+    const shortTitle = p.title.split(/\s+/)[0].toLowerCase().replace(/[^a-z0-9]/g, '');
+    const key = `${familyName}${p.year || '????'}${shortTitle}`;
+
+    const bibData = {
+      key: key,
+      entryType: 'article', 
+      author: p.authors.map(a => a.name).join(' and '),
+      title: p.title,
+      year: p.year ? p.year.toString() : null,
+      venue: p.venue, 
+      booktitle: p.venue,
+      abstract: p.abstract,
+      doi: p.doi,
+      url: p.openAccessPdf ? p.openAccessPdf.url : (p.doi ? `https://doi.org/${p.doi}` : null),
+      google_scholar_id: p.scholarClusterId
+    };
+    
+    const provenance = {
+      wtfp_status: p.doi ? 'official' : 'partial',
+      wtfp_source: p.source,
+      wtfp_citations: p.citationCount,
+      wtfp_velocity: p.wtfp_velocity,
+      wtfp_s2_id: p.paperId,
+      wtfp_scholar_id: p.scholarClusterId
+    };
+
+    return {
+      ...p,
+      bibtex: bibFormat.format(bibData, provenance)
+    };
+  });
+
+  return {
+    results: formatted,
+    metadata: {
+      query,
+      total: unique.length,
+      returned: formatted.length,
+      errors
+    }
+  };
+}
+
+
+// --- CLI Handling ---
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const query = args.find(a => !a.startsWith('--'));
+  
+  if (!query) {
+    console.error('Usage: node citation-fetcher.js "<query>" [--intent=seminal] [--year=2023]');
+    process.exit(1);
+  }
+
+  const intentArg = args.find(a => a.startsWith('--intent='));
+  const yearArg = args.find(a => a.startsWith('--year='));
+  const limitArg = args.find(a => a.startsWith('--limit='));
+
+  const options = {
+    intent: intentArg ? intentArg.split('=')[1] : 'balanced',
+    year: yearArg ? yearArg.split('=')[1] : null,
+    limit: limitArg ? parseInt(limitArg.split('=')[1]) : 10
+  };
+
+  search(query, options).then(result => {
+    // console.log(JSON.stringify(result, null, 2));
+    // For now, output just the bibtex entries as text for easy reading, or JSON?
+    // The previous fetcher output JSON. Let's stick to JSON array of results.
+    console.log(JSON.stringify(result.results, null, 2));
+  }).catch(e => {
+    console.error(e);
+    process.exit(1);
+  });
+}
+
+module.exports = { search };

package/bin/lib/citation-ranker.js

@@ -0,0 +1,133 @@
+/**
+ * Citation Ranker
+ * 
+ * Ranks papers based on multidimensional impact score:
+ * - Citation Count (Log scaled)
+ * - Velocity (Citations per month)
+ * - Recency (Decay over time)
+ * - Venue (Tiered scoring)
+ */
+
+// Venue Tiers
+const VENUE_TIERS = {
+  TIER_1: [
+    // AI/ML
+    "neurips", "nips", "icml", "iclr", "cvpr", "iccv", "eccv", "acl", "emnlp", "naacl",
+    "aaai", "ijcai", "kdd", "www", "sigir", "chi",
+    // Systems
+    "osdi", "sosp", "nsdi", "eurosys", "atc", "fast", "sigcomm", "mobicom",
+    // Journals
+    "nature", "science", "pnas", "cell", "lancet", "nejm", "jama",
+    "ieee transactions", "acm computing surveys", "jmlr", "journal of machine learning research"
+  ],
+  TIER_2: [
+    "coling", "wsdm", "cikm", "pakdd", "ijcnn", "icra", "iros",
+    "middleware", "cloud", "sc", "hpdc", "cluster"
+  ],
+  PREPRINT: [
+    "arxiv", "biorxiv", "medrxiv", "ssrn", "workshop"
+  ]
+};
+
+/**
+ * Score a venue name
+ * @param {string} venue 
+ * @returns {number} 0.0 to 1.0
+ */
+function scoreVenue(venue) {
+  if (!venue) return 0.2;
+  const v = venue.toLowerCase();
+  
+  if (VENUE_TIERS.TIER_1.some(t => v.includes(t))) return 1.0;
+  if (VENUE_TIERS.TIER_2.some(t => v.includes(t))) return 0.7;
+  if (VENUE_TIERS.PREPRINT.some(t => v.includes(t))) return 0.3;
+  
+  return 0.5; // Unknown venue default
+}
+
+/**
+ * Calculate citation velocity
+ * @param {number} citations 
+ * @param {number} year 
+ * @returns {number} Citations per month
+ */
+function calculateVelocity(citations, year) {
+  if (!citations || !year) return 0;
+  const now = new Date();
+  const currentYear = now.getFullYear();
+  const currentMonth = now.getMonth(); // 0-11
+  
+  // Calculate months since publication (assuming Jan 1st of year)
+  // If current year, use months passed. If past year, (diff * 12) + currentMonth.
+  let monthsSince = (currentYear - year) * 12 + currentMonth;
+  
+  // Guard against divide by zero or negative (future dates)
+  monthsSince = Math.max(1, monthsSince);
+  
+  return citations / monthsSince;
+}
+
+/**
+ * Calculate impact score for a paper
+ * @param {Object} paper 
+ * @param {string} intent "seminal" | "recent" | "specific" | "balanced"
+ */
+function calculateScore(paper, intent = "balanced") {
+  const now = new Date().getFullYear();
+  const year = paper.year || now;
+  const age = Math.max(0, now - year);
+  const citations = paper.citationCount || 0;
+  const velocity = calculateVelocity(citations, year);
+  
+  // Normalize factors (approximate ranges)
+  // Citations: log10(100,000) = 5. So score is 0-1.
+  const citationScore = Math.min(1, Math.log10(citations + 1) / 5);
+  
+  // Velocity: 100 cites/month is huge. Normalize to 0-1.
+  const velocityScore = Math.min(1, velocity / 100);
+  
+  // Recency: Linear decay over 10 years. 0 if > 10 years old.
+  const recencyScore = Math.max(0, 1 - (age / 10));
+  
+  const venueScore = scoreVenue(paper.venue);
+  
+  // Weights based on intent
+  const weights = {
+    seminal:  { citation: 0.6, velocity: 0.2, recency: 0.1, venue: 0.1 },
+    recent:   { citation: 0.2, velocity: 0.3, recency: 0.4, venue: 0.1 },
+    specific: { citation: 0.1, velocity: 0.1, recency: 0.1, venue: 0.1 }, // Ranking less important for specific lookup
+    balanced: { citation: 0.4, velocity: 0.3, recency: 0.2, venue: 0.1 }
+  };
+  
+  const w = weights[intent] || weights.balanced;
+  
+  return (
+    citationScore * w.citation +
+    velocityScore * w.velocity +
+    recencyScore * w.recency +
+    venueScore * w.venue
+  );
+}
+
+/**
+ * Rank a list of papers
+ * @param {Array} papers 
+ * @param {string} intent 
+ * @returns {Array} Sorted papers with score attached
+ */
+function rank(papers, intent = "balanced") {
+  const scored = papers.map(p => ({
+    ...p,
+    wtfp_score: calculateScore(p, intent),
+    wtfp_velocity: calculateVelocity(p.citationCount, p.year)
+  }));
+  
+  return scored.sort((a, b) => b.wtfp_score - a.wtfp_score);
+}
+
+module.exports = {
+  rank,
+  calculateScore,
+  calculateVelocity,
+  scoreVenue
+};

package/bin/lib/context-primer.js

@@ -0,0 +1,214 @@
+/**
+ * Context Primer - Section-specific context extraction
+ *
+ * Reduces context burn for long papers by loading only relevant
+ * portions of PROJECT.md, argument-map.md, etc. for each task.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Section type → relevant context files mapping
+ */
+const SECTION_CONTEXT_MAP = {
+  abstract: ['argument-map.md', 'outline.md', 'narrative-arc.md'],
+  introduction: ['argument-map.md', 'narrative-arc.md'],
+  'literature-review': ['argument-map.md'],
+  'related-work': ['argument-map.md'],
+  methods: ['argument-map.md'],
+  results: ['argument-map.md'],
+  discussion: ['argument-map.md', 'narrative-arc.md'],
+  conclusion: ['argument-map.md', 'narrative-arc.md'],
+  // Grant-specific
+  'specific-aims': ['argument-map.md'],
+  background: ['argument-map.md', 'narrative-arc.md'],
+  approach: ['argument-map.md'],
+};
+
+/**
+ * Extract relevant sections from PROJECT.md based on task type
+ * @param {string} projectContent - Full PROJECT.md content
+ * @param {string} taskType - Type of task (plan, write, review, research)
+ * @returns {string} - Filtered PROJECT.md content
+ */
+function filterProject(projectContent, taskType) {
+  if (!projectContent) return '';
+
+  // Always include: What This Is, Core Argument
+  const alwaysSections = ['What This Is', 'Core Argument'];
+
+  // Task-specific sections
+  const taskSections = {
+    plan: ['Requirements', 'Target Audience', 'Constraints'],
+    write: ['Core Argument', 'Target Audience'],
+    review: ['Requirements', 'Core Argument', 'Constraints'],
+    research: ['Core Argument', 'Target Audience', 'Key Decisions'],
+    polish: ['Target Audience'],
+  };
+
+  const needed = [...alwaysSections, ...(taskSections[taskType] || alwaysSections)];
+  return extractSections(projectContent, needed);
+}
+
+/**
+ * Extract relevant structure files for a section type
+ * @param {string} planningDir - Path to .planning directory
+ * @param {string} sectionSlug - Section slug (e.g., 'introduction', 'methods')
+ * @returns {Object} - Map of filename → content
+ */
+function getStructureContext(planningDir, sectionSlug) {
+  const structureDir = path.join(planningDir, 'structure');
+  const result = {};
+
+  // Normalize section slug for lookup
+  const normalized = sectionSlug.toLowerCase().replace(/\d+-/, '').replace(/\s+/g, '-');
+
+  // Find matching context files
+  const relevantFiles = SECTION_CONTEXT_MAP[normalized] || ['argument-map.md'];
+
+  for (const file of relevantFiles) {
+    const filePath = path.join(structureDir, file);
+    if (fs.existsSync(filePath)) {
+      result[file] = fs.readFileSync(filePath, 'utf8');
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Load section-specific context (CONTEXT.md + RESEARCH.md)
+ * @param {string} sectionDir - Path to section directory
+ * @returns {Object} - { context, research }
+ */
+function getSectionContext(sectionDir) {
+  const result = { context: '', research: '' };
+
+  if (!fs.existsSync(sectionDir)) return result;
+
+  const files = fs.readdirSync(sectionDir);
+  for (const f of files) {
+    if (f.endsWith('-CONTEXT.md')) {
+      result.context = fs.readFileSync(path.join(sectionDir, f), 'utf8');
+    }
+    if (f.endsWith('-RESEARCH.md')) {
+      result.research = fs.readFileSync(path.join(sectionDir, f), 'utf8');
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Get prior section summaries relevant to current section
+ * @param {string} sectionsDir - Path to .planning/sections/
+ * @param {number} currentSection - Current section number
+ * @param {number} limit - Max characters to return
+ * @returns {string} - Concatenated relevant summaries
+ */
+function getPriorSummaries(sectionsDir, currentSection, limit = 3000) {
+  if (!fs.existsSync(sectionsDir)) return '';
+
+  const dirs = fs.readdirSync(sectionsDir)
+    .filter(d => {
+      const num = parseInt(d.split('-')[0], 10);
+      return !isNaN(num) && num < currentSection;
+    })
+    .sort();
+
+  let output = '';
+  // Work backwards from most recent, within limit
+  for (const dir of dirs.reverse()) {
+    const dirPath = path.join(sectionsDir, dir);
+    const summaries = fs.readdirSync(dirPath).filter(f => f.endsWith('-SUMMARY.md'));
+
+    for (const s of summaries) {
+      const content = fs.readFileSync(path.join(dirPath, s), 'utf8');
+      // Extract just the header section (up to first ---)
+      const header = content.split('---')[0] || content.slice(0, 500);
+      if (output.length + header.length > limit) return output;
+      output = header + '\n---\n' + output;
+    }
+  }
+
+  return output;
+}
+
+/**
+ * Build complete primed context for an agent spawn
+ * @param {Object} options
+ * @param {string} options.planningDir - Path to .planning/
+ * @param {string} options.sectionDir - Path to section directory
+ * @param {number} options.sectionNumber - Section number
+ * @param {string} options.sectionSlug - Section slug
+ * @param {string} options.taskType - plan|write|review|research|polish
+ * @returns {Object} - All context pieces, filtered for relevance
+ */
+function primeContext({ planningDir, sectionDir, sectionNumber, sectionSlug, taskType }) {
+  const projectPath = path.join(planningDir, 'PROJECT.md');
+  const statePath = path.join(planningDir, 'STATE.md');
+  const roadmapPath = path.join(planningDir, 'ROADMAP.md');
+  const sectionsDir = path.join(planningDir, 'sections');
+
+  const projectContent = fs.existsSync(projectPath)
+    ? fs.readFileSync(projectPath, 'utf8') : '';
+
+  return {
+    project: filterProject(projectContent, taskType),
+    state: fs.existsSync(statePath) ? fs.readFileSync(statePath, 'utf8') : '',
+    roadmap: fs.existsSync(roadmapPath) ? fs.readFileSync(roadmapPath, 'utf8') : '',
+    structure: getStructureContext(planningDir, sectionSlug),
+    section: getSectionContext(sectionDir),
+    priorSummaries: getPriorSummaries(sectionsDir, sectionNumber),
+  };
+}
+
+// --- Helpers ---
+
+/**
+ * Extract named sections from markdown content
+ * @param {string} content - Markdown content
+ * @param {string[]} sectionNames - Section heading names to extract
+ * @returns {string} - Extracted sections joined
+ */
+function extractSections(content, sectionNames) {
+  if (!content) return '';
+
+  const lines = content.split('\n');
+  const extracted = [];
+  let capturing = false;
+  let currentLevel = 0;
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,3})\s+(.+)/);
+
+    if (headingMatch) {
+      const level = headingMatch[1].length;
+      const name = headingMatch[2].trim();
+
+      if (sectionNames.some(s => name.includes(s))) {
+        capturing = true;
+        currentLevel = level;
+        extracted.push(line);
+      } else if (capturing && level <= currentLevel) {
+        capturing = false;
+      } else if (capturing) {
+        extracted.push(line);
+      }
+    } else if (capturing) {
+      extracted.push(line);
+    }
+  }
+
+  return extracted.join('\n');
+}
+
+module.exports = {
+  filterProject,
+  getStructureContext,
+  getSectionContext,
+  getPriorSummaries,
+  primeContext,
+  SECTION_CONTEXT_MAP,
+};