wtf-p 0.3.0 → 0.4.0

package/bin/lib/citation-fetcher.js

@@ -0,0 +1,299 @@
+const https = require('https');
+const querystring = require('querystring');
+const s2 = require('./semantic-scholar');
+const scholar = require('./scholar-lookup');
+const ranker = require('./citation-ranker');
+const bibFormat = require('./bib-format');
+
+/**
+ * WTF-P Citation Fetcher v0.4.0
+ * 
+ * Orchestrates tiered search:
+ * 1. Semantic Scholar (Primary, Free)
+ * 2. SerpAPI (Optional, Seminal/Paid)
+ * 3. CrossRef (Fallback)
+ * 
+ * Usage:
+ *   node citation-fetcher.js "<query>" --intent=<intent> --year=<year>
+ */
+
+// --- CrossRef Fallback ---
+async function searchCrossRef(query, limit = 5) {
+  return new Promise((resolve, reject) => {
+    const params = {
+      query: query,
+      rows: limit,
+      sort: 'relevance',
+      select: 'DOI,title,author,issued,type,container-title,volume,issue,page,abstract'
+    };
+
+    const options = {
+      hostname: 'api.crossref.org',
+      path: `/works?${querystring.stringify(params)}`,
+      method: 'GET',
+      headers: {
+        'User-Agent': 'WTF-P/0.4.0 (citation-expert)'
+      }
+    };
+
+    const req = https.request(options, (res) => {
+      let data = '';
+      res.on('data', c => data += c);
+      res.on('end', () => {
+        if (res.statusCode !== 200) return resolve([]);
+        try {
+          const json = JSON.parse(data);
+          const items = json.message.items || [];
+          resolve(items.map(mapCrossRefToPaper));
+        } catch (e) {
+          resolve([]);
+        }
+      });
+    });
+    
+    req.on('error', () => resolve([]));
+    req.end();
+  });
+}
+
+function mapCrossRefToPaper(item) {
+  const title = item.title ? item.title[0] : 'Untitled';
+  const year = item.issued && item.issued['date-parts'] ? item.issued['date-parts'][0][0] : null;
+  
+  return {
+    source: 'crossref',
+    title: title,
+    year: year,
+    doi: item.DOI,
+    venue: item['container-title'] ? item['container-title'][0] : null,
+    authors: (item.author || []).map(a => ({ name: `${a.family}, ${a.given}` })),
+    abstract: item.abstract ? item.abstract.replace(/<[^>]*>?/gm, '').trim() : null,
+    citationCount: 0, // CrossRef doesn't give citation counts easily
+    externalIds: { DOI: item.DOI }
+  };
+}
+
+// --- Mapper ---
+
+function mapS2ToPaper(item) {
+  return {
+    source: 'semantic_scholar',
+    paperId: item.paperId,
+    title: item.title,
+    year: item.year,
+    venue: item.venue,
+    authors: item.authors || [],
+    citationCount: item.citationCount || 0,
+    abstract: item.abstract,
+    externalIds: item.externalIds || {},
+    doi: (item.externalIds && item.externalIds.DOI) || null,
+    openAccessPdf: item.openAccessPdf
+  };
+}
+
+function mapScholarToPaper(item) {
+  return {
+    source: 'google_scholar',
+    scholarClusterId: item.clusterId,
+    title: item.title,
+    year: item.year,
+    venue: item.venue,
+    authors: item.authors || [],
+    citationCount: item.citationCount || 0,
+    abstract: item.snippet, // Use snippet as abstract fallback
+    externalIds: {},
+    doi: null, // Scholar doesn't reliably give DOIs
+    openAccessPdf: item.link ? { url: item.link } : null
+  };
+}
+
+// --- Deduplication ---
+
+function fingerprint(title, year) {
+  if (!title) return `unknown::${Math.random()}`;
+  const normalized = title.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim();
+  return `${normalized}::${year || '????'}`;
+}
+
+function deduplicatePapers(papers) {
+  const seen = new Map();
+
+  for (const paper of papers) {
+    // Priority: DOI > S2ID > ScholarID > Fingerprint
+    const key = paper.doi 
+      || paper.paperId 
+      || paper.scholarClusterId
+      || fingerprint(paper.title, paper.year);
+
+    if (!seen.has(key)) {
+      seen.set(key, paper);
+    } else {
+      // Merge: prefer richer metadata
+      // S2 usually has best metadata (abstracts, verified authors).
+      // Scholar has best citation counts.
+      // CrossRef has verified DOIs.
+      
+      const existing = seen.get(key);
+      let merged = { ...existing };
+      
+      // If new one is S2, take its metadata but keep higher citation count
+      if (paper.source === 'semantic_scholar') {
+        merged = { ...paper, citationCount: Math.max(existing.citationCount, paper.citationCount) };
+        if (existing.scholarClusterId) merged.scholarClusterId = existing.scholarClusterId;
+      } 
+      // If new one is Scholar, just update citation count and maybe ID
+      else if (paper.source === 'google_scholar') {
+        merged.citationCount = Math.max(merged.citationCount, paper.citationCount);
+        merged.scholarClusterId = paper.clusterId;
+      }
+      
+      seen.set(key, merged);
+    }
+  }
+
+  return Array.from(seen.values());
+}
+
+// --- Main Search Logic ---
+
+async function search(query, options = {}) {
+  const limit = options.limit || 10;
+  const intent = options.intent || 'balanced';
+  
+  let papers = [];
+  const errors = [];
+  
+  const searchPromises = [];
+
+  // 1. S2 Search (Always)
+  searchPromises.push(
+    s2.search(query, { limit: limit * 2, year: options.year })
+      .then(res => res.map(mapS2ToPaper))
+      .catch(e => {
+        errors.push(`S2 Error: ${e.message}`);
+        return [];
+      })
+  );
+  
+  // 2. Scholar Search (Conditional)
+  if ((intent === 'seminal' || options.useScholar) && scholar.isAvailable()) {
+    searchPromises.push(
+      scholar.search(query, { limit: limit, yearLow: options.year, yearHigh: options.year })
+        .then(res => res.map(mapScholarToPaper))
+        .catch(e => {
+          errors.push(`Scholar Error: ${e.message}`);
+          return [];
+        })
+    );
+  }
+
+  const results = await Promise.all(searchPromises);
+  results.forEach(r => papers.push(...r));
+
+  // 3. CrossRef Fallback (if few results)
+  if (papers.length < 5) {
+    try {
+      const crResults = await searchCrossRef(query, limit);
+      papers.push(...crResults);
+    } catch (e) {
+      errors.push(`CrossRef Error: ${e.message}`);
+    }
+  }
+
+  // 4. Deduplicate
+  const unique = deduplicatePapers(papers);
+
+  // 5. Rank
+  const ranked = ranker.rank(unique, intent);
+
+  // 6. Format to BibTeX
+  const formatted = ranked.slice(0, limit).map(p => {
+    // Generate key
+    const firstAuthor = p.authors && p.authors.length > 0 
+      ? (p.authors[0].name ? p.authors[0].name.split(',')[0].trim().split(' ').pop().toLowerCase() : 'unknown') 
+      : 'unknown';
+      
+    // Handle S2 author format variants or string parsing
+    let familyName = 'unknown';
+    if (p.authors && p.authors.length > 0) {
+       const nameParts = p.authors[0].name.split(' ');
+       familyName = nameParts[nameParts.length - 1].toLowerCase().replace(/[^a-z]/g, '');
+    }
+
+    const shortTitle = p.title.split(/\s+/)[0].toLowerCase().replace(/[^a-z0-9]/g, '');
+    const key = `${familyName}${p.year || '????'}${shortTitle}`;
+
+    const bibData = {
+      key: key,
+      entryType: 'article', 
+      author: p.authors.map(a => a.name).join(' and '),
+      title: p.title,
+      year: p.year ? p.year.toString() : null,
+      venue: p.venue, 
+      booktitle: p.venue,
+      abstract: p.abstract,
+      doi: p.doi,
+      url: p.openAccessPdf ? p.openAccessPdf.url : (p.doi ? `https://doi.org/${p.doi}` : null),
+      google_scholar_id: p.scholarClusterId
+    };
+    
+    const provenance = {
+      wtfp_status: p.doi ? 'official' : 'partial',
+      wtfp_source: p.source,
+      wtfp_citations: p.citationCount,
+      wtfp_velocity: p.wtfp_velocity,
+      wtfp_s2_id: p.paperId,
+      wtfp_scholar_id: p.scholarClusterId
+    };
+
+    return {
+      ...p,
+      bibtex: bibFormat.format(bibData, provenance)
+    };
+  });
+
+  return {
+    results: formatted,
+    metadata: {
+      query,
+      total: unique.length,
+      returned: formatted.length,
+      errors
+    }
+  };
+}
+
+
+// --- CLI Handling ---
+
+if (require.main === module) {
+  const args = process.argv.slice(2);
+  const query = args.find(a => !a.startsWith('--'));
+  
+  if (!query) {
+    console.error('Usage: node citation-fetcher.js "<query>" [--intent=seminal] [--year=2023]');
+    process.exit(1);
+  }
+
+  const intentArg = args.find(a => a.startsWith('--intent='));
+  const yearArg = args.find(a => a.startsWith('--year='));
+  const limitArg = args.find(a => a.startsWith('--limit='));
+
+  const options = {
+    intent: intentArg ? intentArg.split('=')[1] : 'balanced',
+    year: yearArg ? yearArg.split('=')[1] : null,
+    limit: limitArg ? parseInt(limitArg.split('=')[1]) : 10
+  };
+
+  search(query, options).then(result => {
+    // console.log(JSON.stringify(result, null, 2));
+    // For now, output just the bibtex entries as text for easy reading, or JSON?
+    // The previous fetcher output JSON. Let's stick to JSON array of results.
+    console.log(JSON.stringify(result.results, null, 2));
+  }).catch(e => {
+    console.error(e);
+    process.exit(1);
+  });
+}
+
+module.exports = { search };

package/bin/lib/citation-ranker.js

@@ -0,0 +1,133 @@
+/**
+ * Citation Ranker
+ * 
+ * Ranks papers based on multidimensional impact score:
+ * - Citation Count (Log scaled)
+ * - Velocity (Citations per month)
+ * - Recency (Decay over time)
+ * - Venue (Tiered scoring)
+ */
+
+// Venue Tiers
+const VENUE_TIERS = {
+  TIER_1: [
+    // AI/ML
+    "neurips", "nips", "icml", "iclr", "cvpr", "iccv", "eccv", "acl", "emnlp", "naacl",
+    "aaai", "ijcai", "kdd", "www", "sigir", "chi",
+    // Systems
+    "osdi", "sosp", "nsdi", "eurosys", "atc", "fast", "sigcomm", "mobicom",
+    // Journals
+    "nature", "science", "pnas", "cell", "lancet", "nejm", "jama",
+    "ieee transactions", "acm computing surveys", "jmlr", "journal of machine learning research"
+  ],
+  TIER_2: [
+    "coling", "wsdm", "cikm", "pakdd", "ijcnn", "icra", "iros",
+    "middleware", "cloud", "sc", "hpdc", "cluster"
+  ],
+  PREPRINT: [
+    "arxiv", "biorxiv", "medrxiv", "ssrn", "workshop"
+  ]
+};
+
+/**
+ * Score a venue name
+ * @param {string} venue 
+ * @returns {number} 0.0 to 1.0
+ */
+function scoreVenue(venue) {
+  if (!venue) return 0.2;
+  const v = venue.toLowerCase();
+  
+  if (VENUE_TIERS.TIER_1.some(t => v.includes(t))) return 1.0;
+  if (VENUE_TIERS.TIER_2.some(t => v.includes(t))) return 0.7;
+  if (VENUE_TIERS.PREPRINT.some(t => v.includes(t))) return 0.3;
+  
+  return 0.5; // Unknown venue default
+}
+
+/**
+ * Calculate citation velocity
+ * @param {number} citations 
+ * @param {number} year 
+ * @returns {number} Citations per month
+ */
+function calculateVelocity(citations, year) {
+  if (!citations || !year) return 0;
+  const now = new Date();
+  const currentYear = now.getFullYear();
+  const currentMonth = now.getMonth(); // 0-11
+  
+  // Calculate months since publication (assuming Jan 1st of year)
+  // If current year, use months passed. If past year, (diff * 12) + currentMonth.
+  let monthsSince = (currentYear - year) * 12 + currentMonth;
+  
+  // Guard against divide by zero or negative (future dates)
+  monthsSince = Math.max(1, monthsSince);
+  
+  return citations / monthsSince;
+}
+
+/**
+ * Calculate impact score for a paper
+ * @param {Object} paper 
+ * @param {string} intent "seminal" | "recent" | "specific" | "balanced"
+ */
+function calculateScore(paper, intent = "balanced") {
+  const now = new Date().getFullYear();
+  const year = paper.year || now;
+  const age = Math.max(0, now - year);
+  const citations = paper.citationCount || 0;
+  const velocity = calculateVelocity(citations, year);
+  
+  // Normalize factors (approximate ranges)
+  // Citations: log10(100,000) = 5. So score is 0-1.
+  const citationScore = Math.min(1, Math.log10(citations + 1) / 5);
+  
+  // Velocity: 100 cites/month is huge. Normalize to 0-1.
+  const velocityScore = Math.min(1, velocity / 100);
+  
+  // Recency: Linear decay over 10 years. 0 if > 10 years old.
+  const recencyScore = Math.max(0, 1 - (age / 10));
+  
+  const venueScore = scoreVenue(paper.venue);
+  
+  // Weights based on intent
+  const weights = {
+    seminal:  { citation: 0.6, velocity: 0.2, recency: 0.1, venue: 0.1 },
+    recent:   { citation: 0.2, velocity: 0.3, recency: 0.4, venue: 0.1 },
+    specific: { citation: 0.1, velocity: 0.1, recency: 0.1, venue: 0.1 }, // Ranking less important for specific lookup
+    balanced: { citation: 0.4, velocity: 0.3, recency: 0.2, venue: 0.1 }
+  };
+  
+  const w = weights[intent] || weights.balanced;
+  
+  return (
+    citationScore * w.citation +
+    velocityScore * w.velocity +
+    recencyScore * w.recency +
+    venueScore * w.venue
+  );
+}
+
+/**
+ * Rank a list of papers
+ * @param {Array} papers 
+ * @param {string} intent 
+ * @returns {Array} Sorted papers with score attached
+ */
+function rank(papers, intent = "balanced") {
+  const scored = papers.map(p => ({
+    ...p,
+    wtfp_score: calculateScore(p, intent),
+    wtfp_velocity: calculateVelocity(p.citationCount, p.year)
+  }));
+  
+  return scored.sort((a, b) => b.wtfp_score - a.wtfp_score);
+}
+
+module.exports = {
+  rank,
+  calculateScore,
+  calculateVelocity,
+  scoreVenue
+};

package/bin/lib/manifest.js

@@ -26,6 +26,24 @@ const MANIFEST = {
         dest: 'skills/wtfp',
         type: 'dir'
       },
+      {
+        id: 'agents',
+        src: path.join(ROOT, 'vendors', 'claude', 'agents', 'wtfp'),
+        dest: 'agents/wtfp',
+        type: 'dir'
+      },
+      {
+        id: 'mcp',
+        src: path.join(ROOT, 'vendors', 'claude', 'mcp'),
+        dest: 'mcp',
+        type: 'dir'
+      },
+      {
+        id: 'scripts',
+        src: path.join(ROOT, 'bin', 'lib'),
+        dest: 'bin',
+        type: 'dir'
+      },
       {
         id: 'plugin',
         src: path.join(ROOT, 'vendors', 'claude', '.claude-plugin'),

package/bin/lib/scholar-lookup.js

@@ -0,0 +1,188 @@
+const https = require('https');
+const querystring = require('querystring');
+
+/**
+ * SerpAPI Google Scholar Wrapper
+ * 
+ * Provides access to Google Scholar data via SerpAPI.
+ * Used for high-value "seminal" queries to get citation velocity and cluster IDs.
+ * 
+ * Env:
+ *  - SERPAPI_KEY: Required for operation.
+ */
+
+const HOST = 'serpapi.com';
+const PATH = '/search';
+
+const usage = {
+  queries: 0,
+  // Default budget: 100 queries/month (~$5)
+  monthlyBudget: 100,
+  
+  canQuery() {
+    return this.queries < this.monthlyBudget;
+  },
+  
+  recordQuery() {
+    this.queries++;
+    if (this.queries % 10 === 0) {
+      // console.error(`[Scholar] API usage: ${this.queries}/${this.monthlyBudget}`);
+    }
+  }
+};
+
+function getApiKey() {
+  return process.env.SERPAPI_KEY;
+}
+
+function isAvailable() {
+  return !!getApiKey();
+}
+
+function request(params) {
+  return new Promise((resolve, reject) => {
+    const key = getApiKey();
+    if (!key) {
+      return reject(new Error('SERPAPI_KEY not set'));
+    }
+    if (!usage.canQuery()) {
+      return reject(new Error('Monthly budget exceeded'));
+    }
+
+    usage.recordQuery();
+
+    const q = querystring.stringify({
+      ...params,
+      api_key: key,
+      engine: 'google_scholar'
+    });
+
+    const options = {
+      hostname: HOST,
+      path: `${PATH}?${q}`,
+      method: 'GET'
+    };
+
+    const req = https.request(options, (res) => {
+      let data = '';
+      res.on('data', c => data += c);
+      res.on('end', () => {
+        if (res.statusCode !== 200) {
+          return reject(new Error(`SerpAPI Error: ${res.statusCode} ${data}`));
+        }
+        try {
+          resolve(JSON.parse(data));
+        } catch (e) {
+          reject(e);
+        }
+      });
+    });
+
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+/**
+ * Search Google Scholar
+ * @param {string} query 
+ * @param {Object} options { yearLow, yearHigh, limit }
+ */
+async function search(query, options = {}) {
+  const params = {
+    q: query,
+    num: options.limit || 10
+  };
+  
+  if (options.yearLow) params.as_ylo = options.yearLow;
+  if (options.yearHigh) params.as_yhi = options.yearHigh;
+
+  const data = await request(params);
+  return (data.organic_results || []).map(mapResult);
+}
+
+/**
+ * Get paper by Cluster ID
+ * @param {string} clusterId 
+ */
+async function getByClusterId(clusterId) {
+  const data = await request({ cluster: clusterId });
+  // When searching by cluster, it returns list of versions.
+  // We usually want the first one or metadata about the cluster.
+  // SerpAPI "cluster" usually returns "organic_results" which are the versions.
+  const results = data.organic_results || [];
+  if (results.length > 0) return mapResult(results[0]);
+  return null;
+}
+
+/**
+ * Get citing papers
+ * @param {string} clusterId 
+ * @param {number} limit 
+ */
+async function getCitingPapers(clusterId, limit = 10) {
+  const data = await request({ 
+    cites: clusterId,
+    num: limit 
+  });
+  return (data.organic_results || []).map(mapResult);
+}
+
+function mapResult(item) {
+  // Extract cluster ID
+  let clusterId = null;
+  // Try to find it in links or specific fields if SerpAPI provides it directly
+  // SerpAPI usually puts it in inline_links -> cited_by -> serpapi_link (cites=ID)
+  // or versions -> serpapi_link (cluster=ID)
+  
+  if (item.inline_links && item.inline_links.cited_by && item.inline_links.cited_by.serpapi_link) {
+    const match = item.inline_links.cited_by.serpapi_link.match(/cites=(\w+)/);
+    if (match) clusterId = match[1];
+  }
+  
+  return {
+    source: 'google_scholar',
+    title: item.title,
+    clusterId: clusterId || item.cluster_id, // Sometimes provided directly
+    link: item.link,
+    snippet: item.snippet,
+    publication_info: item.publication_info,
+    citationCount: item.inline_links && item.inline_links.cited_by ? item.inline_links.cited_by.total : 0,
+    year: extractYear(item.publication_info ? item.publication_info.summary : ''),
+    authors: extractAuthors(item.publication_info ? item.publication_info.summary : ''),
+    venue: extractVenue(item.publication_info ? item.publication_info.summary : '')
+  };
+}
+
+function extractYear(summary) {
+  const match = summary.match(/\b(19|20)\d{2}\b/);
+  return match ? parseInt(match[0]) : null;
+}
+
+function extractAuthors(summary) {
+  // "A Author, B Author - Venue, 2020 - publisher"
+  if (!summary) return [];
+  const parts = summary.split('-');
+  if (parts.length > 0) {
+    return parts[0].split(',').map(s => ({ name: s.trim() }));
+  }
+  return [];
+}
+
+function extractVenue(summary) {
+  if (!summary) return null;
+  const parts = summary.split('-');
+  if (parts.length > 1) {
+    // "Venue, Year" usually
+    return parts[1].split(',')[0].trim();
+  }
+  return null;
+}
+
+module.exports = {
+  isAvailable,
+  search,
+  getByClusterId,
+  getCitingPapers,
+  usage
+};