thinking-phrases 1.0.1 → 2.0.0

package/src/core/config.ts

@@ -28,35 +28,50 @@ export function resolveConfigPath(configPath?: string): string {
 
 export const DEFAULT_CONFIG: Config = {
   feeds: [],
-  limit: 25,
+  rssFetchIntervalSeconds: 21600,
+  limit: 100,
   mode: 'replace',
   target: 'auto',
   phraseFormatting: {
     includeSource: true,
     includeTime: true,
     maxLength: 140,
+    templates: {
+      article: '%title% — %source% (%time%)',
+      hackerNews: '%title% — HN %score% (%time%)',
+      stock: '%symbol% %price% %change% %market%',
+      githubCommit: '%headline% — %repo% %delta% @%author% (%time%)',
+      githubFeed: '%action% — @%handle% (%time%)',
+    },
   },
   githubModels: {
     enabled: false,
-    model: 'openai/gpt-4.1',
+    endpoint: 'https://models.github.ai/inference',
+    model: 'openai/gpt-4.1-mini',
     tokenEnvVar: 'GITHUB_MODELS_TOKEN',
     maxInputItems: 10,
-    maxTokens: 300,
+    maxInputTokens: 16000,
+    maxTokens: 500,
+    maxConcurrency: 1,
     maxPhrasesPerArticle: 2,
     temperature: 0.2,
     fetchArticleContent: true,
     maxArticleContentLength: 6000,
+    cacheTtlSeconds: 604800,
   },
   stockQuotes: {
     enabled: false,
     symbols: ['MSFT', 'NVDA', 'TSLA', 'AMZN', 'GOOGL', 'AMD'],
     includeMarketState: true,
+    showClosed: false,
+    fetchIntervalSeconds: 60,
   },
   hackerNews: {
     enabled: false,
     feed: 'top',
     maxItems: 10,
     minScore: 50,
+    fetchIntervalSeconds: 300,
   },
   earthquakes: {
     enabled: false,
@@ -66,6 +81,7 @@ export const DEFAULT_CONFIG: Config = {
     limit: 10,
     radiusKm: 500,
     orderBy: 'time',
+    fetchIntervalSeconds: 1800,
   },
   weatherAlerts: {
     enabled: false,
@@ -73,6 +89,7 @@ export const DEFAULT_CONFIG: Config = {
     area: '',
     minimumSeverity: 'moderate',
     limit: 10,
+    fetchIntervalSeconds: 1800,
   },
   customJson: {
     enabled: false,
@@ -86,6 +103,7 @@ export const DEFAULT_CONFIG: Config = {
     dateField: 'publishedAt',
     idField: 'id',
     maxItems: 10,
+    fetchIntervalSeconds: 3600,
   },
   githubActivity: {
     enabled: false,
@@ -98,6 +116,7 @@ export const DEFAULT_CONFIG: Config = {
     maxItems: 10,
     sinceHours: 24,
     tokenEnvVar: 'GITHUB_TOKEN',
+    fetchIntervalSeconds: 300,
   },
 };
 
@@ -272,6 +291,12 @@ export function parseArgs(argv: string[]): CliOverrides {
           index += 1;
         }
         break;
+      case '--models-max-input-tokens':
+        if (next) {
+          setModels({ maxInputTokens: Number(next) });
+          index += 1;
+        }
+        break;
       case '--models-max-tokens':
         if (next) {
           setModels({ maxTokens: Number(next) });
@@ -290,6 +315,18 @@ export function parseArgs(argv: string[]): CliOverrides {
           index += 1;
         }
         break;
+      case '--models-endpoint':
+        if (next) {
+          setModels({ endpoint: next });
+          index += 1;
+        }
+        break;
+      case '--models-max-concurrency':
+        if (next) {
+          setModels({ maxConcurrency: Number(next) });
+          index += 1;
+        }
+        break;
       case '--fetch-article-content':
         setModels({ fetchArticleContent: true });
         break;
@@ -566,6 +603,7 @@ export function mergeConfig(base: Config, fileConfig: Partial<Config>, argConfig
     verbose: argConfig.verbose ?? fileConfig.verbose ?? base.verbose,
     debug: argConfig.debug ?? fileConfig.debug ?? base.debug,
     feeds: argConfig.feeds ?? fileConfig.feeds ?? base.feeds,
+    rssFetchIntervalSeconds: argConfig.rssFetchIntervalSeconds ?? fileConfig.rssFetchIntervalSeconds ?? base.rssFetchIntervalSeconds,
     phraseFormatting: {
       ...base.phraseFormatting,
       ...(fileConfig.phraseFormatting ?? {}),
@@ -601,6 +639,7 @@ export function mergeConfig(base: Config, fileConfig: Partial<Config>, argConfig
       ...(fileConfig.customJson ?? {}),
       ...(argConfig.customJson ?? {}),
     },
+    customJsonSources: argConfig.customJsonSources ?? fileConfig.customJsonSources ?? base.customJsonSources,
     githubActivity: {
       ...base.githubActivity,
       ...(fileConfig.githubActivity ?? {}),
@@ -617,6 +656,7 @@ export function validateConfig(config: Config): void {
     && !config.earthquakes.enabled
     && !config.weatherAlerts.enabled
     && !config.customJson.enabled
+    && !(config.customJsonSources ?? []).some(s => s.enabled)
     && !config.githubActivity.enabled
   ) {
     throw new Error('Configure at least one source before running dynamic phrases.');
@@ -647,6 +687,14 @@ export function validateConfig(config: Config): void {
     throw new Error(`githubModels.temperature must be between 0 and 1. Received: ${config.githubModels.temperature}`);
   }
 
+  if (!Number.isFinite(config.githubModels.maxConcurrency) || config.githubModels.maxConcurrency < 1) {
+    throw new Error(`githubModels.maxConcurrency must be at least 1. Received: ${config.githubModels.maxConcurrency}`);
+  }
+
+  if (config.githubModels.endpoint && !/^https?:\/\//u.test(config.githubModels.endpoint)) {
+    throw new Error(`githubModels.endpoint must be a valid HTTP(S) URL. Received: ${config.githubModels.endpoint}`);
+  }
+
   const invalidFeed = config.feeds.find(feed => !feed.url.trim());
   if (invalidFeed) {
     throw new Error('Every feed entry must include a non-empty url.');
@@ -682,6 +730,20 @@ export function validateConfig(config: Config): void {
     }
   }
 
+  for (const [index, source] of (config.customJsonSources ?? []).entries()) {
+    if (!source.enabled) {
+      continue;
+    }
+
+    if (!source.url.trim()) {
+      throw new Error(`customJsonSources[${index}].url must be set when enabled.`);
+    }
+
+    if (!source.titleField.trim()) {
+      throw new Error(`customJsonSources[${index}].titleField must be set when enabled.`);
+    }
+  }
+
   if (config.githubActivity.enabled) {
     if (config.githubActivity.mode === 'repo-commits' && !config.githubActivity.repo?.trim()) {
       throw new Error('githubActivity.repo must be set when githubActivity.mode is repo-commits.');

package/src/core/githubModels.ts

@@ -1,14 +1,114 @@
-import ModelClient, { isUnexpected } from '@azure-rest/ai-inference';
-import { AzureKeyCredential } from '@azure/core-auth';
+import OpenAI from 'openai';
 import { execFileSync } from 'node:child_process';
-import type { ArticleItem, Config, GitHubModelsConfig, GitHubModelsResponse } from './types.js';
+import type { ArticleItem, Config, GitHubModelsConfig } from './types.js';
 import { decodeHtmlEntities, dedupePhrases, logDebug, singleLine } from './utils.js';
+import { appendSourceSuffix } from './phraseFormats.js';
 
 interface BuildModelArticlePhrasesOptions {
   onProgress?: (message: string) => void;
+  /** Called with each batch of phrases as they're generated */
+  onPhrases?: (phrases: string[]) => void;
+  /** Source type for prompt selection (rss, hacker-news, github-activity, etc.) */
+  sourceType?: string;
 }
 
-const GITHUB_MODELS_ENDPOINT = 'https://models.github.ai/inference';
+const PROMPT_PREAMBLE = [
+  'IMPORTANT: Your ENTIRE response must be valid JSON and nothing else. No markdown, no explanation, no code fences.',
+  'Return exactly this shape: {"phrases":["phrase1","phrase2"]}.',
+  'Each phrase: factual, concrete, max maxLength chars. You may emit up to maxPhrasesPerArticle phrases per item.',
+  'NEVER include the source name, author, date, or time — those are appended automatically.',
+  'NEVER restate the title verbatim. The reader already saw the headline — give them the insight behind it.',
+  'Each phrase is displayed INDEPENDENTLY — never start with "It", "This", "The project", "The tool", or any pronoun that refers to something the reader hasn\'t seen.',
+  'Every phrase must be self-contained and make sense on its own without context from other phrases.',
+].join(' ');
+
+export const DEFAULT_SOURCE_PROMPTS: Record<string, string> = {
+  'rss': [
+    PROMPT_PREAMBLE,
+    'You are extracting insights from blog posts and news articles (RSS/Atom feeds).',
+    'You receive the article title AND the full article body text.',
+    'Your job: find the single most valuable, concrete takeaway buried in the article that the reader would NOT get from the title alone.',
+    'Prioritize: specific numbers, benchmarks, percentages, technical details, surprising findings, release dates, breaking changes, or "how it works" explanations.',
+    'BAD: "GitHub released a new feature for code review" (just restates the headline).',
+    'GOOD: "Copilot code review now uses multi-line comments that reduced cognitive load by 15% in A/B testing".',
+    'BAD: "The article discusses improvements to Docker performance".',
+    'GOOD: "Docker BuildKit v0.17 parallelizes dependency resolution, cutting cold builds from 4m to 90s on large monorepos".',
+    'If the article body has real data, use it. If it is too thin, extract the most specific claim from the title and sharpen it.',
+  ].join(' '),
+  'hacker-news': [
+    PROMPT_PREAMBLE,
+    'You are extracting insights from Hacker News posts. You may receive:',
+    '(a) The HN title + the full linked article body (most common — link posts)',
+    '(b) The HN title + the self-post text (Ask HN, Show HN)',
+    '(c) The HN title + both the self-post text AND fetched article body',
+    'Your job: extract the ONE technical insight, surprising fact, or concrete detail that makes this post worth reading.',
+    'The reader has 3 seconds of glance time. Make it count with a real fact, not a summary.',
+    'BAD: "A database was built in a spreadsheet" (just restates the HN title).',
+    'GOOD: "The spreadsheet-database uses SQLite compiled to WASM, handling 10k rows with indexed queries under 50ms".',
+    'BAD: "The author discusses their experience with Rust".',
+    'GOOD: "Switching from Go to Rust cut their p99 latency from 12ms to 800μs by eliminating GC pauses".',
+    'For Show HN posts: what does it actually do and what makes it technically interesting?',
+    'For Ask HN posts: what is the most insightful or surprising answer/claim?',
+  ].join(' '),
+  'github-activity': [
+    PROMPT_PREAMBLE,
+    'You are summarizing GitHub commits. You receive the commit message AND the full diff (added/removed lines).',
+    'Your job: explain the PURPOSE and IMPACT of the change in plain language. What is different for users or developers AFTER this commit?',
+    'Read the diff carefully — the commit message often undersells the change. The diff tells the real story.',
+    'BAD: "Fixed a null check in the settings handler" (says what, not why).',
+    'GOOD: "Settings panel no longer crashes when opening a workspace with a corrupted .vscode/settings.json".',
+    'BAD: "Refactored the entrypoint module".',
+    'GOOD: "DevTools now loads 40% faster after the entrypoint was split into lazy-loaded chunks".',
+    'For performance changes: include the before/after numbers if visible in the diff.',
+    'For bug fixes: describe the user-visible symptom that was fixed.',
+    'For new features: describe what users can now do that they couldn\'t before.',
+    'NEVER mention file paths, line counts, or SHA hashes — those appear in the metadata suffix.',
+  ].join(' '),
+  'earthquakes': [
+    PROMPT_PREAMBLE,
+    'You are summarizing USGS earthquake data. You receive magnitude, location, significance score, alert level, and tsunami status.',
+    'Keep it concise and factual. The magnitude and location are already in the title — add context that helps the reader understand the severity.',
+    'If significance is high (>500) or an alert level is set, emphasize that.',
+    'If a tsunami bulletin was issued, lead with that.',
+    'BAD: "An earthquake happened near Ridgecrest" (obvious from the title).',
+    'GOOD: "Significance 680 with yellow alert — strongest quake in the region since the 2019 Ridgecrest sequence".',
+    'If the data is sparse (just magnitude + location with no alert), a clean one-liner with the depth or felt radius is fine.',
+  ].join(' '),
+  'custom-json': [
+    PROMPT_PREAMBLE,
+    'You are summarizing items from a custom JSON API. The data structure varies.',
+    'Extract the most concrete, specific, and informative detail from each item.',
+    'Focus on facts the reader can learn in a glance: numbers, names, outcomes, technical details.',
+    'BAD: "An interesting article about cloud computing".',
+    'GOOD: "AWS Lambda now supports 10GB memory functions, enabling in-memory ML inference without containers".',
+  ].join(' '),
+};
+
+const DEFAULT_FALLBACK_PROMPT = [
+  PROMPT_PREAMBLE,
+  'Extract the single most valuable, concrete takeaway from each item.',
+  'Prioritize: specific numbers, technical details, surprising findings, or "what changed and why it matters".',
+  'The reader has 3 seconds. Give them a real insight, not a headline restatement.',
+].join(' ');
+
+/**
+ * Resolve the prompt for a given source type.
+ * Priority: config per-source prompt > config systemPrompt > built-in per-source default > built-in fallback.
+ */
+export function resolvePrompt(config: GitHubModelsConfig, sourceType?: string): string {
+  if (sourceType && config.prompts?.[sourceType]) {
+    return config.prompts[sourceType];
+  }
+  if (config.systemPrompt) {
+    return config.systemPrompt;
+  }
+  if (sourceType && DEFAULT_SOURCE_PROMPTS[sourceType]) {
+    return DEFAULT_SOURCE_PROMPTS[sourceType];
+  }
+  return DEFAULT_FALLBACK_PROMPT;
+}
+
+const DEFAULT_ENDPOINT = 'https://models.github.ai/inference';
 
 function getGitHubModelsToken(config: GitHubModelsConfig): string | undefined {
   const envToken = process.env[config.tokenEnvVar] ?? process.env.GITHUB_TOKEN;
@@ -20,6 +120,13 @@ function getGitHubModelsToken(config: GitHubModelsConfig): string | undefined {
     const token = execFileSync('gh', ['auth', 'token'], {
       encoding: 'utf8',
       stdio: ['ignore', 'pipe', 'ignore'],
+      env: {
+        ...process.env,
+        GITHUB_TOKEN: '',
+        GH_TOKEN: '',
+        GITHUB_ENTERPRISE_TOKEN: '',
+        GH_ENTERPRISE_TOKEN: '',
+      },
     }).trim();
 
     return token || undefined;
@@ -28,7 +135,7 @@ function getGitHubModelsToken(config: GitHubModelsConfig): string | undefined {
   }
 }
 
-function extractModelPhrases(input: string): string[] {
+export function extractModelPhrases(input: string): string[] {
   const candidate = (input.match(/```(?:json)?\s*([\s\S]*?)```/u)?.[1] ?? input).trim();
 
   try {
@@ -65,7 +172,7 @@ function extractModelPhrases(input: string): string[] {
     .split(/\r?\n/u)
     .map(line => line.trim())
     .filter(Boolean)
-    .map(line => line.replace(/^[\[\]",*-•\s]+/gu, '').trim())
+    .map(line => line.replace(/^[[\]",*\-•\s]+/gu, '').trim())
     .filter(Boolean);
 }
 
@@ -77,75 +184,64 @@ async function runGitHubModelsPrompt(config: GitHubModelsConfig, content: string
     );
   }
 
-  const client = ModelClient(GITHUB_MODELS_ENDPOINT, new AzureKeyCredential(token));
-  const response = await client.path('/chat/completions').post({
-    body: {
-      model: config.model,
-      messages: [{ role: 'user', content }],
-      temperature: config.temperature,
-      max_tokens: config.maxTokens,
-      response_format: { type: 'json_object' },
-    },
+  const client = new OpenAI({
+    baseURL: config.endpoint || DEFAULT_ENDPOINT,
+    apiKey: token,
   });
 
-  if (isUnexpected(response)) {
-    const errorBody = response.body as { error?: { message?: string } };
-    throw new Error(errorBody.error?.message ?? 'GitHub Models request failed.');
-  }
+  // Reasoning models (o1, o3, o4, gpt-5) don't support temperature or response_format
+  const isReasoningModel = /(?:^|\/)(?:o[1-4]|gpt-5)/iu.test(config.model);
+
+  // Reasoning models need more tokens because thinking tokens count against the budget
+  const maxTokens = isReasoningModel ? Math.max(config.maxTokens * 4, 2000) : config.maxTokens;
 
-  const text = (response.body as GitHubModelsResponse).choices?.[0]?.message?.content?.trim();
+  const completion = await client.chat.completions.create({
+    model: config.model,
+    messages: [{ role: 'user', content }],
+    ...(!isReasoningModel && config.temperature !== 1 ? { temperature: config.temperature } : {}),
+    max_completion_tokens: maxTokens,
+    ...(!isReasoningModel ? { response_format: { type: 'json_object' } } : {}),
+  });
+
+  const text = completion.choices?.[0]?.message?.content?.trim();
   if (!text) {
-    throw new Error('GitHub Models response did not include content.');
+    const reason = completion.choices?.[0]?.finish_reason;
+    throw new Error(`GitHub Models returned empty content (finish_reason: ${reason ?? 'unknown'})`);
   }
 
   return text;
 }
 
-function chunkArticles(articles: ArticleItem[], config: GitHubModelsConfig): ArticleItem[][] {
-  const chunks: ArticleItem[][] = [];
-  const estimatedPerChunk = Math.max(1, Math.floor(config.maxTokens / Math.max(80, config.maxPhrasesPerArticle * 80)));
-  const defaultChunkSize = Math.max(1, Math.min(config.maxInputItems, estimatedPerChunk));
-  const maxCharactersPerChunk = 24_000;
-
-  let currentChunk: ArticleItem[] = [];
-  let currentCharacters = 0;
-
-  const estimateArticleCharacters = (article: ArticleItem): number => {
-    return [article.title, article.source, article.time, article.articleContent, article.content, article.link]
-      .filter(Boolean)
-      .join(' ')
-      .length;
-  };
-
-  const flushCurrentChunk = (): void => {
-    if (currentChunk.length > 0) {
-      chunks.push(currentChunk);
-      currentChunk = [];
-      currentCharacters = 0;
-    }
-  };
-
-  for (const article of articles) {
-    const articleCharacters = estimateArticleCharacters(article);
-
-    if (
-			currentChunk.length > 0
-			&& (currentChunk.length >= defaultChunkSize || currentCharacters + articleCharacters > maxCharactersPerChunk)
-		) {
-      flushCurrentChunk();
-    }
+async function summarizeArticle(article: ArticleItem, config: Config, sourceType?: string): Promise<string[]> {
+  const instruction = resolvePrompt(config.githubModels, sourceType);
+  const payload = JSON.stringify({
+    instruction,
+    maxLength: config.phraseFormatting.maxLength,
+    maxPhrasesPerArticle: config.githubModels.maxPhrasesPerArticle,
+    items: [{
+      title: article.title ?? '',
+      source: article.source ?? '',
+      time: article.time ?? '',
+      content: (article.articleContent ?? article.content ?? '').slice(0, config.githubModels.maxArticleContentLength),
+      link: article.link ?? '',
+    }],
+  });
 
-    currentChunk.push(article);
-    currentCharacters += articleCharacters;
+  logDebug(config, `Sending "${article.title}" (${payload.length} chars) to GitHub Models`);
+  const responseText = await runGitHubModelsPrompt(config.githubModels, payload);
+  logDebug(config, `Response: ${singleLine(responseText, 220)}`);
 
-    if (articleCharacters > maxCharactersPerChunk) {
-      flushCurrentChunk();
-    }
-  }
+  return extractModelPhrases(responseText)
+    .map(phrase => singleLine(decodeHtmlEntities(phrase), config.phraseFormatting.maxLength))
+    .filter(Boolean)
+    .map(phrase => appendSourceSuffix(phrase, article.source, article.time, article.metadata));
+}
 
-  flushCurrentChunk();
+const DELAY_BETWEEN_REQUESTS_MS = 2000;
+const MAX_RETRIES = 3;
 
-  return chunks;
+async function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
 }
 
 export async function buildModelArticlePhrases(
@@ -153,56 +249,48 @@ export async function buildModelArticlePhrases(
   config: Config,
   options: BuildModelArticlePhrasesOptions = {},
 ): Promise<string[]> {
-  const chunks = chunkArticles(articles, config.githubModels);
-  let completedChunks = 0;
-
-  options.onProgress?.(`Generating phrases with GitHub Models (${chunks.length} batch${chunks.length === 1 ? '' : 'es'} in parallel)`);
-
-  const settledChunkResults = await Promise.allSettled(
-    chunks.map(async (chunk, index) => {
-      const payload = JSON.stringify({
-        instruction: config.githubModels.systemPrompt ?? [
-          'Create concise VS Code thinking phrases from these normalized content items.',
-          'Return JSON only in this shape: {"phrases":["..."]}.',
-          'Each phrase must be factual, concrete, and at most maxLength characters.',
-          'You may emit multiple phrases for one item when it has multiple distinct takeaways.',
-          'Return at most maxPhrasesPerArticle phrases per item.',
-          'Prefer concrete details like numbers, locations, dates, features, examples, or outcomes.',
-          'Avoid vague rewrites of the headline.',
-        ].join(' '),
-        maxLength: config.phraseFormatting.maxLength,
-        maxPhrasesPerArticle: config.githubModels.maxPhrasesPerArticle,
-        items: chunk.map(article => ({
-          title: article.title ?? '',
-          source: article.source ?? '',
-          time: article.time ?? '',
-          content: article.articleContent ?? article.content ?? '',
-          link: article.link ?? '',
-        })),
-      });
-
-      logDebug(config, `Sending ${chunk.length} items to GitHub Models for chunk ${index + 1}/${chunks.length}`);
-      const responseText = await runGitHubModelsPrompt(config.githubModels, payload);
-      logDebug(config, `Model response preview: ${singleLine(responseText, 220)}`);
-
-      completedChunks += 1;
-      options.onProgress?.(`Generated GitHub Models phrases (${completedChunks}/${chunks.length})`);
-
-      return extractModelPhrases(responseText)
-        .map(phrase => singleLine(decodeHtmlEntities(phrase), config.phraseFormatting.maxLength))
-        .filter(Boolean);
-    }),
-  );
-
-  const rejectedChunk = settledChunkResults.find(
-    (result): result is PromiseRejectedResult => result.status === 'rejected',
-  );
-
-  if (rejectedChunk) {
-    throw rejectedChunk.reason;
+  options.onProgress?.(`Generating phrases with GitHub Models (${articles.length} article${articles.length === 1 ? '' : 's'})`);
+
+  const allPhrases: string[] = [];
+  let lastError: unknown;
+
+  for (let i = 0; i < articles.length; i++) {
+    const article = articles[i];
+    const title = article.title ?? 'untitled';
+    options.onProgress?.(`Summarizing (${i + 1}/${articles.length}): ${title.length > 60 ? title.slice(0, 60) + '…' : title}`);
+    let phrases: string[] | undefined;
+
+    for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        phrases = await summarizeArticle(article, config, options.sourceType);
+        break;
+      } catch (error: unknown) {
+        const is429 = error instanceof Error && error.message.includes('429');
+        if (is429 && attempt < MAX_RETRIES) {
+          const backoff = attempt * 5000;
+          options.onProgress?.(`Rate limited — waiting ${backoff / 1000}s before retry (${attempt}/${MAX_RETRIES})`);
+          await sleep(backoff);
+          continue;
+        }
+        lastError = error;
+        const message = error instanceof Error ? error.message : String(error);
+        logDebug(config, `Failed "${article.title}": ${message}`);
+        options.onProgress?.(`GitHub Models failed (${i + 1}/${articles.length}): ${message}`);
+        break;
+      }
+    }
+
+    if (phrases && phrases.length > 0) {
+      allPhrases.push(...phrases);
+      options.onPhrases?.(phrases);
+    }
+
+    options.onProgress?.(`Generated phrases (${i + 1}/${articles.length})`);
+  }
+
+  if (allPhrases.length === 0 && lastError) {
+    throw lastError;
   }
 
-  return dedupePhrases(
-    settledChunkResults.flatMap(result => (result.status === 'fulfilled' ? result.value : [])),
-  );
+  return dedupePhrases(allPhrases);
 }