thinking-phrases 1.0.1 → 2.0.0

package/src/sources/githubActivity.ts

@@ -1,6 +1,7 @@
 import { execFileSync } from 'node:child_process';
-import type { ArticleItem, Config, GitHubActivityConfig, GitHubFeedKind, PhraseSource } from '../core/types.js';
-import { fetchJson, fetchText, logDebug, logInfo, relativeTime, singleLine, truncate } from '../core/utils.js';
+import { formatGitHubCommitPhrase, formatGitHubFeedPhrase } from '../core/phraseFormats.js';
+import type { ArticleItem, Config, GitHubActivityConfig, GitHubFeedKind, PhraseFormatTemplates, PhraseSource } from '../core/types.js';
+import { USER_AGENT, fetchJson, fetchText, hoursToMs, logDebug, logInfo, relativeTime, singleLine } from '../core/utils.js';
 import { hydrateArticleContent, parseFeedArticles } from './rss.js';
 
 interface GitHubCommitListItem {
@@ -24,14 +25,14 @@ interface GitHubCommitDetail extends GitHubCommitListItem {
     deletions?: number;
     total?: number;
   };
-  files?: Array<{
+  files?: {
     filename?: string;
     status?: string;
     additions?: number;
     deletions?: number;
     changes?: number;
     patch?: string;
-  }>;
+  }[];
 }
 
 interface GitHubCommitContext {
@@ -93,17 +94,30 @@ interface GitHubAuthenticatedUser {
 
 const GITHUB_ACCEPT = 'application/vnd.github+json';
 const GITHUB_API_VERSION = '2022-11-28';
+let hasWarnedAboutRejectedGitHubToken = false;
+let hasWarnedAboutInvalidConfiguredToken = false;
 
-function getGitHubToken(config: GitHubActivityConfig): string | undefined {
-  const envToken = process.env[config.tokenEnvVar] ?? process.env.GITHUB_TOKEN;
-  if (envToken && !envToken.includes('replace_me')) {
-    return envToken;
-  }
+interface GitHubTokenCandidate {
+  source: string;
+  token: string;
+}
 
+function isPlaceholderToken(token?: string): boolean {
+  return !token || token.includes('replace_me');
+}
+
+function getGitHubCliToken(): string | undefined {
   try {
     const token = execFileSync('gh', ['auth', 'token'], {
       encoding: 'utf8',
       stdio: ['ignore', 'pipe', 'ignore'],
+      env: {
+        ...process.env,
+        GITHUB_TOKEN: '',
+        GH_TOKEN: '',
+        GITHUB_ENTERPRISE_TOKEN: '',
+        GH_ENTERPRISE_TOKEN: '',
+      },
     }).trim();
 
     return token || undefined;
@@ -112,6 +126,61 @@ function getGitHubToken(config: GitHubActivityConfig): string | undefined {
   }
 }
 
+function getGitHubTokenCandidates(config: GitHubActivityConfig): GitHubTokenCandidate[] {
+  const candidates: GitHubTokenCandidate[] = [];
+  const seen = new Set<string>();
+
+  const addCandidate = (source: string, token?: string): void => {
+    if (isPlaceholderToken(token) || !token || seen.has(token)) {
+      return;
+    }
+
+    seen.add(token);
+    candidates.push({ source, token });
+  };
+
+  addCandidate(config.tokenEnvVar, process.env[config.tokenEnvVar]);
+  if (config.tokenEnvVar !== 'GITHUB_TOKEN') {
+    addCandidate('GITHUB_TOKEN', process.env.GITHUB_TOKEN);
+  }
+  addCandidate('gh auth token', getGitHubCliToken());
+
+  return candidates;
+}
+
+async function validateGitHubToken(token: string): Promise<boolean> {
+  try {
+    const response = await fetch('https://api.github.com/user', {
+      headers: {
+        ...buildGitHubHeaders(token),
+        'user-agent': USER_AGENT,
+      },
+      signal: AbortSignal.timeout(15_000),
+    });
+
+    return response.ok;
+  } catch {
+    return false;
+  }
+}
+
+async function getGitHubToken(config: GitHubActivityConfig): Promise<string | undefined> {
+  const candidates = getGitHubTokenCandidates(config);
+
+  for (const candidate of candidates) {
+    if (await validateGitHubToken(candidate.token)) {
+      if (candidate.source === 'gh auth token' && candidates[0] && candidates[0].source !== 'gh auth token' && !hasWarnedAboutInvalidConfiguredToken) {
+        hasWarnedAboutInvalidConfiguredToken = true;
+        console.warn(`Configured ${candidates[0].source} was invalid; using GitHub CLI auth instead`);
+      }
+
+      return candidate.token;
+    }
+  }
+
+  return undefined;
+}
+
 function buildGitHubHeaders(token?: string, accept = GITHUB_ACCEPT): Record<string, string> {
   return {
     accept,
@@ -121,7 +190,16 @@ function buildGitHubHeaders(token?: string, accept = GITHUB_ACCEPT): Record<stri
 }
 
 function isGitHubAuthFailure(error: unknown): boolean {
-  return error instanceof Error && /\((401|403)\)/u.test(error.message);
+  return error instanceof Error && /\(401\)/u.test(error.message);
+}
+
+function warnRejectedGitHubTokenOnce(): void {
+  if (hasWarnedAboutRejectedGitHubToken) {
+    return;
+  }
+
+  hasWarnedAboutRejectedGitHubToken = true;
+  console.warn('GitHub token rejected, retrying without auth');
 }
 
 async function fetchGitHubJson<T>(url: string, token?: string): Promise<T> {
@@ -129,7 +207,7 @@ async function fetchGitHubJson<T>(url: string, token?: string): Promise<T> {
     return await fetchJson<T>(url, buildGitHubHeaders(token));
   } catch (error) {
     if (token && isGitHubAuthFailure(error)) {
-      console.warn('GitHub token rejected, retrying without auth');
+      warnRejectedGitHubTokenOnce();
       return fetchJson<T>(url, buildGitHubHeaders(undefined));
     }
 
@@ -139,7 +217,7 @@ async function fetchGitHubJson<T>(url: string, token?: string): Promise<T> {
 
 async function fetchGitHubJsonWithHeaders<T>(url: string, headers: Record<string, string>): Promise<T> {
   return fetchJson<T>(url, {
-    'user-agent': 'thinking-phrases/1.0 (+https://github.com/austenstone/thinking-phrases)',
+    'user-agent': USER_AGENT,
     ...headers,
   });
 }
@@ -149,7 +227,7 @@ async function fetchGitHubText(url: string, token?: string, accept = 'applicatio
     return await fetchText(url, buildGitHubHeaders(token, accept));
   } catch (error) {
     if (token && isGitHubAuthFailure(error)) {
-      console.warn('GitHub token rejected, retrying without auth');
+      warnRejectedGitHubTokenOnce();
       return fetchText(url, buildGitHubHeaders(undefined, accept));
     }
 
@@ -207,11 +285,7 @@ function repoDisplayName(repoLabel: string): string {
   return parts.at(-1) ?? trimmed;
 }
 
-function buildCommitContent(detail: GitHubCommitDetail, config: Config): string | undefined {
-  return buildCommitContentFromContext({ detail }, config);
-}
-
-function buildCommitContentFromContext(context: GitHubCommitContext, config: Config): string | undefined {
+function buildCommitContentFromContext(context: GitHubCommitContext, _config: Config): string | undefined {
   const { detail, diffText } = context;
   const files = detail.files ?? [];
   const summaryBits = [
@@ -253,29 +327,22 @@ function buildShortShaLabel(detail: GitHubCommitDetail): string | undefined {
   return sha ? sha.slice(0, 7) : undefined;
 }
 
-function buildCommitDisplayPhrase(repoLabel: string, detail: GitHubCommitDetail): string {
-  const repoName = repoDisplayName(repoLabel);
-  const shortShaLabel = buildShortShaLabel(detail);
-  const commitTitle = firstCommitLine(detail.commit?.message);
-  const deltaLabel = buildCommitDeltaLabel(detail);
-  const authorHandle = detail.author?.login?.trim();
-  const time = relativeTime(detail.commit?.author?.date);
-
-  const headline = `${repoName}${shortShaLabel ? `@${shortShaLabel}` : ''} ${commitTitle}`.trim();
-  const metadata = [
-    deltaLabel ? `(${deltaLabel})` : undefined,
-    time,
-    authorHandle ? `- @${authorHandle}` : undefined,
-  ].filter(Boolean);
-
-  return [headline, ...metadata].join(' ');
+function buildCommitDisplayPhrase(repoLabel: string, detail: GitHubCommitDetail, templates?: PhraseFormatTemplates): string {
+  return formatGitHubCommitPhrase({
+    headline: firstCommitLine(detail.commit?.message),
+    delta: buildCommitDeltaLabel(detail),
+    repo: repoDisplayName(repoLabel),
+    sha: buildShortShaLabel(detail),
+    author: detail.author?.login?.trim(),
+    time: relativeTime(detail.commit?.author?.date),
+  }, { template: templates?.githubCommit });
 }
 
 function escapeRegExp(input: string): string {
   return input.replace(/[.*+?^${}()|[\]\\]/gu, '\\$&');
 }
 
-function buildGitHubFeedDisplayPhrase(article: ArticleItem): string | undefined {
+function buildGitHubFeedDisplayPhrase(article: ArticleItem, templates?: PhraseFormatTemplates): string | undefined {
   const title = article.title?.trim();
   if (!title) {
     return undefined;
@@ -286,16 +353,15 @@ function buildGitHubFeedDisplayPhrase(article: ArticleItem): string | undefined
   const maybeHandle = source && source !== 'GitHub' ? source : undefined;
 
   if (!maybeHandle) {
-    return time ? `${title} — ${time}` : title;
+    return formatGitHubFeedPhrase({ action: title, time }, { template: templates?.githubFeed });
   }
 
   // Strip the actor name from the start of the title if present (exact or case-insensitive)
   const handlePattern = new RegExp(`^${escapeRegExp(maybeHandle)}\\s*`, 'iu');
   const strippedTitle = title.replace(handlePattern, '').trim();
   const actionText = strippedTitle || title;
-  const finalTitle = `@${maybeHandle} ${actionText}`;
 
-  return time ? `${finalTitle} — ${time}` : finalTitle;
+  return formatGitHubFeedPhrase({ handle: maybeHandle, action: actionText, time }, { template: templates?.githubFeed });
 }
 
 function repoNameFromEvent(repo?: string): string {
@@ -353,7 +419,7 @@ function buildOrganizationEventLink(event: GitHubOrgEvent): string | undefined {
     ?? event.payload?.release?.html_url;
 }
 
-function buildOrganizationEventArticle(event: GitHubOrgEvent): ArticleItem | null {
+function buildOrganizationEventArticle(event: GitHubOrgEvent, templates?: PhraseFormatTemplates): ArticleItem | null {
   const title = buildOrganizationEventTitle(event);
   if (!title) {
     return null;
@@ -373,11 +439,11 @@ function buildOrganizationEventArticle(event: GitHubOrgEvent): ArticleItem | nul
 
   return {
     ...article,
-    displayPhrase: buildGitHubFeedDisplayPhrase(article),
+    displayPhrase: buildGitHubFeedDisplayPhrase(article, templates),
   };
 }
 
-function buildCommitArticle(repoLabel: string, detail: GitHubCommitDetail, content?: string): ArticleItem {
+function buildCommitArticle(repoLabel: string, detail: GitHubCommitDetail, content?: string, templates?: PhraseFormatTemplates): ArticleItem {
   const datetime = detail.commit?.author?.date;
   const repoName = repoDisplayName(repoLabel);
   const authorHandle = detail.author?.login?.trim();
@@ -394,13 +460,19 @@ function buildCommitArticle(repoLabel: string, detail: GitHubCommitDetail, conte
     type: 'article',
     id: `github:${repoLabel}:${detail.sha}`,
     title: titleBits.join(' — '),
-		displayPhrase: buildCommitDisplayPhrase(repoLabel, detail),
+		displayPhrase: buildCommitDisplayPhrase(repoLabel, detail, templates),
     link: detail.html_url,
     source: repoName,
     datetime,
     time: relativeTime(datetime),
     content: detail.commit?.message?.trim(),
     articleContent: content,
+    metadata: {
+      repo: repoName,
+      delta: deltaLabel,
+      author: authorHandle ? `@${authorHandle}` : undefined,
+      sha: shortShaLabel,
+    },
   };
 }
 
@@ -433,10 +505,10 @@ async function fetchCommitContext(owner: string, repo: string, ref: string, conf
 
 async function fetchRepoCommitArticles(config: Config): Promise<ArticleItem[]> {
   const { owner, repo } = parseRepoSlug(config.githubActivity.repo);
-  const token = getGitHubToken(config.githubActivity);
+  const token = await getGitHubToken(config.githubActivity);
   const params = new URLSearchParams({
     per_page: String(config.githubActivity.maxItems),
-    since: new Date(Date.now() - config.githubActivity.sinceHours * 60 * 60 * 1000).toISOString(),
+    since: new Date(Date.now() - hoursToMs(config.githubActivity.sinceHours)).toISOString(),
   });
 
   if (config.githubActivity.branch?.trim()) {
@@ -462,12 +534,12 @@ async function fetchRepoCommitArticles(config: Config): Promise<ArticleItem[]> {
 }
 
 async function fetchOrgCommitArticles(config: Config): Promise<ArticleItem[]> {
-  const token = getGitHubToken(config.githubActivity);
+  const token = await getGitHubToken(config.githubActivity);
   const params = new URLSearchParams({ per_page: String(Math.min(Math.max(config.githubActivity.maxItems * 10, 50), 100)) });
   const eventsUrl = `https://api.github.com/orgs/${config.githubActivity.org}/events?${params.toString()}`;
   logInfo(config, `Fetching GitHub org events from ${eventsUrl}`);
   const events = await fetchGitHubJson<GitHubOrgEvent[]>(eventsUrl, token);
-  const sinceThreshold = Date.now() - config.githubActivity.sinceHours * 60 * 60 * 1000;
+  const sinceThreshold = Date.now() - hoursToMs(config.githubActivity.sinceHours);
 
   const pushEvents = events
     .filter(event => event.type === 'PushEvent' && event.repo?.name && event.payload?.head)
@@ -538,7 +610,7 @@ async function resolveFeedUrl(config: Config, token?: string): Promise<string> {
 }
 
 async function fetchGitHubFeedArticles(config: Config): Promise<ArticleItem[]> {
-  const token = getGitHubToken(config.githubActivity);
+  const token = await getGitHubToken(config.githubActivity);
   let feedUrl: string | undefined;
   try {
     feedUrl = await resolveFeedUrl(config, token);
@@ -552,7 +624,7 @@ async function fetchGitHubFeedArticles(config: Config): Promise<ArticleItem[]> {
     logInfo(config, `Falling back to GitHub organization events from ${eventsUrl}`);
     const events = await fetchGitHubJson<GitHubOrgEvent[]>(eventsUrl, token);
     return events
-      .map(buildOrganizationEventArticle)
+      .map((event) => buildOrganizationEventArticle(event))
       .filter((article): article is ArticleItem => Boolean(article))
       .slice(0, config.githubActivity.maxItems);
   }

package/src/sources/hackerNews.ts

@@ -1,6 +1,9 @@
-import type { ArticleItem, PhraseSource } from '../core/types.js';
+import { formatHackerNewsPhrase } from '../core/phraseFormats.js';
+import type { ArticleItem, Config, PhraseSource } from '../core/types.js';
 import { fetchJson, logInfo, relativeTime, stripHtml } from '../core/utils.js';
 
+const HN_API_BASE = 'https://hacker-news.firebaseio.com/v0';
+
 interface HackerNewsItem {
   by?: string;
   descendants?: number;
@@ -26,20 +29,20 @@ function buildHackerNewsLink(id: number, explicitUrl?: string): string {
   return explicitUrl?.trim() || `https://news.ycombinator.com/item?id=${id}`;
 }
 
-export async function fetchHackerNewsArticles(config: import('../core/types.js').Config): Promise<ArticleItem[]> {
+export async function fetchHackerNewsArticles(config: Config): Promise<ArticleItem[]> {
   if (!config.hackerNews.enabled) {
     return [];
   }
 
   const listName = HN_ENDPOINTS[config.hackerNews.feed];
-  const ids = await fetchJson<number[]>(`https://hacker-news.firebaseio.com/v0/${listName}.json`);
+  const ids = await fetchJson<number[]>(`${HN_API_BASE}/${listName}.json`);
   const fetchCount = Math.min(Math.max(config.hackerNews.maxItems * 3, config.hackerNews.maxItems), 60);
   const candidateIds = ids.slice(0, fetchCount);
 
   logInfo(config, `Fetching ${candidateIds.length} Hacker News items from ${config.hackerNews.feed}`);
 
   const rawItems = await Promise.all(
-    candidateIds.map(id => fetchJson<HackerNewsItem>(`https://hacker-news.firebaseio.com/v0/item/${id}.json`).catch(() => null)),
+    candidateIds.map(id => fetchJson<HackerNewsItem>(`${HN_API_BASE}/item/${id}.json`).catch(() => null)),
   );
 
   return rawItems
@@ -51,7 +54,11 @@ export async function fetchHackerNewsArticles(config: import('../core/types.js')
       const trimmedTitle = item.title?.trim() ?? '';
       const relativeTimestamp = relativeTime(datetime);
       const score = typeof item.score === 'number' ? `+${item.score}` : undefined;
-      const displayPhrase = [`HN: ${trimmedTitle}`, score, relativeTimestamp].filter(Boolean).join(' — ');
+      const displayPhrase = formatHackerNewsPhrase(
+        { title: trimmedTitle, score, time: relativeTimestamp },
+        { template: config.phraseFormatting.templates?.hackerNews },
+      );
+      const hnText = stripHtml(item.text);
 
       return {
         type: 'article' as const,
@@ -62,8 +69,13 @@ export async function fetchHackerNewsArticles(config: import('../core/types.js')
         source: 'Hacker News',
         datetime,
         time: relativeTimestamp,
-        content: stripHtml(item.text),
-        articleContent: stripHtml(item.text),
+        content: hnText,
+        articleContent: hnText,
+        metadata: {
+          author: item.by,
+          score: typeof item.score === 'number' ? `${item.score} pts` : undefined,
+          comments: typeof item.descendants === 'number' ? `${item.descendants}` : undefined,
+        },
       };
     });
 }

package/src/sources/rss.ts

@@ -1,6 +1,6 @@
 import { XMLParser } from 'fast-xml-parser';
 import type { ArticleItem, Config, FeedConfig, PhraseSource } from '../core/types.js';
-import { fetchText, logDebug, logInfo, relativeTime, singleLine, stripHtml, truncate } from '../core/utils.js';
+import { decodeHtmlEntities, fetchText, logDebug, logInfo, relativeTime, singleLine, stripHtml, truncate } from '../core/utils.js';
 
 type XmlPrimitive = string | number | boolean | null | undefined;
 
@@ -8,13 +8,14 @@ interface XmlObject {
   [key: string]: XmlValue;
 }
 
-interface XmlArray extends Array<XmlValue> {}
+type XmlArray = XmlValue[];
 
 type XmlValue = XmlPrimitive | XmlObject | XmlArray;
 
 function readText(value: XmlValue): string | undefined {
   if (typeof value === 'string') {
-    return value.trim() || undefined;
+    const decoded = decodeHtmlEntities(value).trim();
+    return decoded || undefined;
   }
 
   if (typeof value === 'number' || typeof value === 'boolean') {
@@ -26,7 +27,12 @@ function readText(value: XmlValue): string | undefined {
   }
 
   const textValue = value['#text'];
-  return typeof textValue === 'string' ? textValue.trim() || undefined : undefined;
+  if (typeof textValue !== 'string') {
+    return undefined;
+  }
+
+  const decoded = decodeHtmlEntities(textValue).trim();
+  return decoded || undefined;
 }
 
 function readLink(value: XmlValue): string | undefined {
@@ -205,27 +211,35 @@ export async function hydrateArticleContent(articles: ArticleItem[], config: Con
   logInfo(config, `Fetching article content for up to ${articles.length} articles`);
 
   return Promise.all(articles.map(async article => {
-    if (article.articleContent) {
-      logDebug(config, `Using embedded feed article content for: ${article.title ?? article.link}`);
+    if (!article.link) {
       return article;
     }
 
-    if (!article.link) {
+    // If the article already has substantial embedded content (e.g. RSS <content:encoded>),
+    // skip fetching — the feed itself provided the full article body.
+    if (article.articleContent && article.articleContent.length >= 500) {
+      logDebug(config, `Using embedded article content (${article.articleContent.length} chars) for: ${article.title ?? article.link}`);
       return article;
     }
 
     try {
       logInfo(config, `Fetching article content from ${article.link}`);
       const html = await fetchText(article.link);
-      const articleContent = extractArticleTextFromHtml(html, config.githubModels.maxArticleContentLength);
+      const fetched = extractArticleTextFromHtml(html, config.githubModels.maxArticleContentLength);
 
-      if (articleContent) {
-        logDebug(config, `Fetched article content preview: ${singleLine(articleContent, 220)}`);
+      if (fetched) {
+        logDebug(config, `Fetched article content preview: ${singleLine(fetched, 220)}`);
       }
 
+      // Combine existing content (e.g. HN self-text) with fetched article body
+      const existing = article.articleContent?.trim();
+      const combined = existing && fetched
+        ? `${existing}\n\n${fetched}`
+        : fetched || existing || article.content;
+
       return {
         ...article,
-        articleContent: articleContent || article.content,
+        articleContent: combined ? truncate(combined, config.githubModels.maxArticleContentLength) : article.content,
       };
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);

package/src/sources/stocks.ts

@@ -1,6 +1,7 @@
 import YahooFinance from 'yahoo-finance2';
+import { formatStockPhrase as formatStockPhraseTemplate } from '../core/phraseFormats.js';
 import type { Config, PhraseSource, StockItem } from '../core/types.js';
-import { dedupePhrases, formatPrice, formatSignedPercent, logInfo, normalizeSymbols, truncate } from '../core/utils.js';
+import { formatPrice, formatSignedPercent, logInfo, normalizeSymbols, truncate } from '../core/utils.js';
 
 interface StockQuoteSnapshot {
   symbol?: string;
@@ -16,6 +17,21 @@ interface StockQuoteSnapshot {
 
 const yahooFinance = new YahooFinance({ suppressNotices: ['yahooSurvey'] });
 
+function formatMarketLabel(label?: string): string | undefined {
+  switch (label) {
+    case 'today':
+      return '🟢';
+    case 'close':
+      return '🔒';
+    case 'pre-market':
+      return '🌅';
+    case 'after-hours':
+      return '🌙';
+    default:
+      return label;
+  }
+}
+
 function getMarketPriceDetails(quote: StockQuoteSnapshot): {
   label?: string;
   price?: number;
@@ -46,17 +62,18 @@ function getMarketPriceDetails(quote: StockQuoteSnapshot): {
 }
 
 export function buildStockPhrase(item: StockItem, config: Config): string {
-  const parts = [item.symbol, formatPrice(item.price, item.currency)];
   const signedPercent = formatSignedPercent(item.changePercent);
-  if (signedPercent) {
-    parts.push(signedPercent);
-  }
+  const formattedMarketLabel = formatMarketLabel(item.marketLabel);
 
-  if (config.stockQuotes.includeMarketState && item.marketLabel) {
-    parts.push(item.marketLabel);
-  }
-
-  return truncate(parts.join(' — '), config.phraseFormatting.maxLength);
+  return truncate(
+    formatStockPhraseTemplate({
+      symbol: item.symbol,
+      price: formatPrice(item.price, item.currency),
+      change: signedPercent,
+      market: config.stockQuotes.includeMarketState ? formattedMarketLabel : undefined,
+    }, { template: config.phraseFormatting.templates?.stock }),
+    config.phraseFormatting.maxLength,
+  );
 }
 
 export async function fetchStockItems(config: Config): Promise<StockItem[]> {
@@ -93,6 +110,10 @@ export async function fetchStockItems(config: Config): Promise<StockItem[]> {
       return [];
     }
 
+    if (!config.stockQuotes.showClosed && quote.marketState === 'CLOSED') {
+      return [];
+    }
+
     const details = getMarketPriceDetails(quote);
     if (!Number.isFinite(details.price)) {
       return [];