sc-research 1.0.6 → 1.0.8

package/README.md

@@ -58,7 +58,6 @@ Template architecture:
    - `research`: `sc-research research`
    - `research:deep`: `sc-research research --depth=deep`
    - `visualize`: `sc-research visualize`
-   - `test:fixed`: `sc-research test:fixed`
 
 2. **Open the project in Claude Code**
 
@@ -72,7 +71,6 @@ Template architecture:
    - `/controversy` – Build controversy map
    - `/discovery` – Discover viral topics
    - `/visualize` – Launch dashboard
-   - `/test-research` – Run fixed-link debug test
 
 ## Direct Bun CLI (Optional, existing behavior)
 

package/dist/cli.js

@@ -223,7 +223,7 @@ var windsurfAdapter = {
     const config = loadPlatformConfig("windsurf");
     return templates.map((template) => renderMarkdownOnly(template, {
       rootDir: config.folderStructure.root,
-      commandDirName: "commands",
+      commandDirName: "workflows",
       skillDirName: "skills",
       skillFileName: "SKILL.md"
     }));
@@ -440,8 +440,7 @@ function syncPackageJson(projectRoot, options) {
   const desiredScripts = {
     research: "sc-research research",
     "research:deep": "sc-research research --depth=deep",
-    visualize: "sc-research visualize",
-    "test:fixed": "sc-research test:fixed"
+    visualize: "sc-research visualize"
   };
   let pkg;
   if (fs4.existsSync(packageJsonPath)) {
@@ -522,10 +521,6 @@ async function main() {
     await runPackagedEntry("visualize.js", rest);
     return;
   }
-  if (command === "test:fixed") {
-    await runPackagedEntry("manual-fixed-links.js", rest);
-    return;
-  }
   const logger = new Logger;
   logger.exitWithError(`Unknown command "${command}". Run "sc-research --help" for usage.`);
 }
@@ -591,7 +586,6 @@ Commands:
   init         Initialize SC-Research support files for a project
   research     Run research engine (same as project "research" script)
   visualize    Launch visualization app
-  test:fixed   Run fixed-link test flow
 
 Options:
   --ai TARGETS         Target AI(s): claude,cursor,windsurf,antigravity or "all"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sc-research",
-  "version": "1.0.6",
+  "version": "1.0.8",
   "description": "Headless Social Media Research Data Provider for AI Agents",
   "type": "module",
   "main": "dist/index.js",
@@ -20,15 +20,13 @@
     "init": "node dist/cli.js init",
     "init:all": "node dist/cli.js init --ai all",
     "visualize": "bun --env-file=.sc-research src/entries/visualize.ts",
-    "test:fixed": "bun --env-file=.sc-research src/test/manual-fixed-links.ts",
     "test:x": "bun --env-file=.sc-research src/test/manual-xai.ts",
     "build:index": "bun build src/entries/research.ts --outfile dist/index.js --target node",
     "build:visualize": "bun build src/entries/visualize.ts --outdir dist --target node",
-    "build:test:fixed": "bun build src/test/manual-fixed-links.ts --outdir dist/test --target node",
     "build:cli": "bun build src/entries/cli.ts --outdir dist --target node",
     "build:web": "npm --prefix web run build",
     "build:web:copy": "node scripts/copy-web-dist.mjs",
-    "build": "bun run build:index && bun run build:visualize && bun run build:test:fixed && bun run build:cli && bun run build:web && bun run build:web:copy",
+    "build": "bun run build:index && bun run build:visualize && bun run build:cli && bun run build:web && bun run build:web:copy",
     "prepack": "bun run build",
     "release:check": "npm pack --dry-run",
     "prepublishOnly": "npm run release:check"

package/templates/base/commands/deep-research.md

@@ -19,7 +19,7 @@ description: Deeply research a topic and route to the best analysis template. Th
     >
     > - Match the requested topic (or close variant of the same topic).
     > - Match requested date window when `--from/--to` was provided.
-    > - Include required fields from `web/src/types.ts`.
+    > - Include required fields from `../skills/social_media_schema/SKILL.md`.
 
 4. Display the results
     > Present whichever template output(s) were selected after validation.

package/templates/base/manifest.json

@@ -7,6 +7,12 @@
       "description": "Worker skill that fetches raw discussion data from Reddit and X (Twitter) for a given topic. Returns raw JSON files.",
       "bodyFile": "skills/social_media_fetch.md"
     },
+    {
+      "id": "social_media_schema",
+      "kind": "skill",
+      "description": "Reference-only skill containing canonical output schemas for classified JSON files.",
+      "bodyFile": "skills/social_media_schema.md"
+    },
     {
       "id": "social_media_rank",
       "kind": "skill",
@@ -43,12 +49,6 @@
       "description": "Worker skill that launches a local web dashboard to visualize all available classified research data.",
       "bodyFile": "skills/social_media_visualize.md"
     },
-    {
-      "id": "social_media_research_test",
-      "kind": "skill",
-      "description": "Test the social media research skill with fixed Reddit links (no API key needed). Fetches data and returns JSON for AI classification.",
-      "bodyFile": "skills/social_media_research_test.md"
-    },
     {
       "id": "using_social_media_research",
       "kind": "skill",
@@ -108,12 +108,6 @@
       "kind": "command",
       "description": "Launch dashboard for available classified data files.",
       "bodyFile": "commands/visualize.md"
-    },
-    {
-      "id": "test-research",
-      "kind": "command",
-      "description": "Run fixed-link test research pipeline.",
-      "bodyFile": "commands/test-research.md"
     }
   ]
 }

package/templates/base/skills/social_media_controversy.md

@@ -25,7 +25,7 @@ At least one valid source file must exist.
 
 ## Step 2: Lock Output Schema
 
-Read `web/src/types.ts` and treat `ControversyData` as source of truth.
+Read `../social_media_schema/SKILL.md` and treat `ControversyData` as source of truth.
 
 Required top-level fields:
 
@@ -91,4 +91,4 @@ Save strict JSON to:
 1. **No external fetch**: analyze existing data only.
 2. **No fabricated arguments**: only report controversies present in source text.
 3. **No fabricated citations**: quote text, author, and link must be real.
-4. **Schema strictness**: if instructions conflict with `types.ts`, `types.ts` wins.
+4. **Schema strictness**: if instructions conflict with schema reference, schema reference wins.

package/templates/base/skills/social_media_discovery.md

@@ -25,7 +25,7 @@ At least one valid source file must exist.
 
 ## Step 2: Lock Schema
 
-Read `web/src/types.ts` and follow `DiscoveryData` and `DiscoveryTopic` exactly.
+Read `../social_media_schema/SKILL.md` and follow `DiscoveryData` and `DiscoveryTopic` exactly.
 
 Important enum constraints:
 
@@ -87,5 +87,5 @@ Save strict JSON to:
 
 1. **No external fetch**: do not run data collection here.
 2. **No fabricated clusters or quotes**: everything must map to real evidence.
-3. **Schema strictness**: `types.ts` is authoritative.
+3. **Schema strictness**: the schema reference skill is authoritative.
 4. **Graceful fallback**: use empty arrays for missing optional evidence; never invent content.

package/templates/base/skills/social_media_rank.md

@@ -29,11 +29,11 @@ If both sources are missing or invalid, stop and report the failure.
 
 ## Step 2: Lock Output Schema
 
-Read `web/src/types.ts` and treat it as source of truth.
+Read `../social_media_schema/SKILL.md` and treat it as source of truth.
 
 - Output type must match `ClassifiedData`.
 - Product entries must match `Product`.
-- If this file and `types.ts` conflict, `types.ts` wins.
+- If this file and schema reference conflict, schema reference wins.
 
 ## Step 3: Build Product Candidates
 

package/templates/base/skills/social_media_schema.md

@@ -0,0 +1,167 @@
+---
+name: social_media_schema
+description: Reference-only skill that defines canonical JSON schemas for classified output files.
+---
+
+# Social Media Schema Reference
+
+Use this file as the canonical schema source for all classified outputs:
+
+- `classified_rank.json`
+- `classified_sentiment.json`
+- `classified_trend.json`
+- `classified_controversy.json`
+- `classified_discovery.json`
+
+If another skill instruction conflicts with this file, this file wins.
+
+## Canonical Type Definitions
+
+```typescript
+export interface ClassifiedData {
+    topic: string;
+    source_file?: string;
+    products: Product[];
+    key_insights: string[];
+}
+
+export interface Product {
+    rank: number;
+    name: string;
+    sentiment: SentimentLabel;
+    mentions: number;
+    estimated_engagement_score: number;
+    consensus: string;
+    pros: string[];
+    cons: string[];
+    highlight_quotes: Array<{
+        text: string;
+        author: string;
+        link: string;
+        context?: "pro" | "con" | "general";
+    }>;
+}
+
+export type SentimentLabel =
+    | "Positive"
+    | "Negative"
+    | "Mixed"
+    | "Very Positive";
+
+export interface SentimentData {
+    topic: string;
+    overall_mood: SentimentLabel;
+    distribution: {
+        very_positive: number;
+        positive: number;
+        mixed: number;
+        negative: number;
+    };
+    by_source: {
+        reddit: SourceSentiment;
+        x: SourceSentiment;
+    };
+    product_sentiments: ProductSentiment[];
+}
+
+export interface SourceSentiment {
+    very_positive: number;
+    positive: number;
+    mixed: number;
+    negative: number;
+}
+
+export interface ProductSentiment {
+    name: string;
+    overall: SentimentLabel;
+    reddit_sentiment: SentimentLabel | null;
+    x_sentiment: SentimentLabel | null;
+    evidence_quotes: Array<{
+        text: string;
+        author: string;
+        link: string;
+        sentiment: SentimentLabel;
+    }>;
+}
+
+export interface TrendData {
+    topic: string;
+    date_range: {
+        from: string;
+        to: string;
+    };
+    granularity?: "day" | "week" | "month";
+    timeline: TimelinePoint[];
+    key_moments: KeyMoment[];
+}
+
+export interface TimelinePoint {
+    period: string;
+    post_count: number;
+    total_engagement: number;
+    reddit_posts: number;
+    x_posts: number;
+}
+
+export interface KeyMoment {
+    date: string;
+    event: string;
+    significance: "high" | "medium" | "low";
+    url?: string;
+}
+
+export interface ControversyData {
+    topic: string;
+    overall_divisiveness: "Low" | "Medium" | "High";
+    controversies: Controversy[];
+}
+
+export interface Controversy {
+    topic: string;
+    heat_score: number;
+    divisiveness: "Low" | "Medium" | "High";
+    side_a: ControversySide;
+    side_b: ControversySide;
+}
+
+export interface ControversySide {
+    position: string;
+    supporter_count: number;
+    sample_quotes: Array<{
+        text: string;
+        author: string;
+        link: string;
+    }>;
+}
+
+export interface DiscoveryData {
+    topic: string;
+    period: string;
+    total_posts_analyzed: number;
+    trending_topics: DiscoveryTopic[];
+}
+
+export interface DiscoveryTopic {
+    id: string;
+    topic_name: string;
+    description: string;
+    category: string;
+    engagement_score: number;
+    sentiment: "positive" | "negative" | "neutral" | "mixed";
+    key_posts: KeyPost[];
+    highlight_comments: Array<{
+        text: string;
+        author: string;
+        link: string;
+        platform: "reddit" | "x";
+    }>;
+}
+
+export interface KeyPost {
+    title: string;
+    url: string;
+    platform: "reddit" | "x";
+    engagement: number;
+    thumbnail?: string;
+}
+```

package/templates/base/skills/social_media_sentiment.md

@@ -25,7 +25,7 @@ At least one valid source file must exist.
 
 ## Step 2: Lock Schema and Allowed Labels
 
-Read `web/src/types.ts` before building output.
+Read `../social_media_schema/SKILL.md` before building output.
 
 - Output must match `SentimentData`.
 - Allowed labels are only:
@@ -89,5 +89,5 @@ Save result to:
 
 1. **No external fetch**: analyze only provided raw files.
 2. **Evidence over guesswork**: every product label must be explainable from quotes/content.
-3. **Strict schema**: `types.ts` is the only schema source of truth.
+3. **Strict schema**: the schema reference skill is the only schema source of truth.
 4. **No fabricated citations**: quote text, author, and link must exist in raw data.

package/templates/base/skills/social_media_trend.md

@@ -67,7 +67,7 @@ Prefer evidence-backed moments over generic commentary.
 
 ## Step 5: Write Output
 
-Read `web/src/types.ts` and output strict `TrendData` JSON to:
+Read `../social_media_schema/SKILL.md` and output strict `TrendData` JSON to:
 
 - `classified_trend.json`
 
@@ -83,5 +83,5 @@ Read `web/src/types.ts` and output strict `TrendData` JSON to:
 
 1. **No external fetch**: analyze existing raw files only.
 2. **No invented dates/events**: every bucket and key moment must come from real data.
-3. **Schema strictness**: `types.ts` is authoritative.
+3. **Schema strictness**: the schema reference skill is authoritative.
 4. **Graceful degradation**: skip bad records; never crash the whole report for a few malformed items.

package/templates/platforms/agent.json

@@ -1,6 +1,6 @@
 {
   "platform": "antigravity",
   "folderStructure": {
-    "root": ".agent"
+    "root": ".agents"
   }
 }

package/dist/test/manual-fixed-links.js

@@ -1,186 +0,0 @@
-import { createRequire } from "node:module";
-var __require = /* @__PURE__ */ createRequire(import.meta.url);
-
-// src/core/clients/reddit.ts
-var DEFAULT_CONFIG = {
-  userAgent: "CommunitiesResearchSkill/1.0.0 (Research)",
-  maxThreads: 20,
-  delayMs: 2000
-};
-var sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-async function fetchRedditThread(threadUrl, config = {}) {
-  const { userAgent } = { ...DEFAULT_CONFIG, ...config };
-  let jsonUrl = threadUrl.split("?")[0];
-  if (jsonUrl.endsWith("/")) {
-    jsonUrl = jsonUrl.slice(0, -1);
-  }
-  jsonUrl += ".json?raw_json=1";
-  let attempts = 0;
-  const maxRetries = 3;
-  while (attempts < maxRetries) {
-    attempts++;
-    try {
-      const response = await fetch(jsonUrl, {
-        headers: {
-          "User-Agent": userAgent || `ResearchSkill/${Date.now()}`,
-          Accept: "application/json",
-          "Accept-Language": "en-US,en;q=0.9"
-        },
-        redirect: "follow"
-      });
-      if (!response.ok) {
-        if (response.status === 404) {
-          console.warn(`[404] Thread not found: ${jsonUrl}`);
-          return [];
-        }
-        if (response.status === 403) {
-          if (attempts === 1) {
-            console.warn(`[403] Blocked. Waiting 5s before single retry...`);
-            await sleep(5000);
-            continue;
-          }
-          throw new Error(`Reddit Blocked (403): Access denied.`);
-        }
-        if (response.status === 429) {
-          const waitTime = Math.pow(2, attempts) * 1000 + Math.random() * 1000;
-          console.warn(`[429] Rate limit. Backing off for ${Math.round(waitTime)}ms...`);
-          await sleep(waitTime);
-          continue;
-        }
-        if (response.status >= 500) {
-          const waitTime = 1000 * attempts;
-          console.warn(`[${response.status}] Server error. Retrying in ${waitTime}ms...`);
-          await sleep(waitTime);
-          continue;
-        }
-        throw new Error(`Failed to fetch Reddit thread: ${response.statusText} (${response.status})`);
-      }
-      const data = await response.json();
-      if (!Array.isArray(data) || data.length < 2) {
-        return [];
-      }
-      const postListing = data[0].data.children[0]?.data;
-      const commentListing = data[1].data.children;
-      if (!postListing)
-        return [];
-      const posts = [];
-      posts.push({
-        text: postListing.title + `
-
-` + (postListing.selftext || ""),
-        title: postListing.title,
-        author: postListing.author,
-        engagement: {
-          score: postListing.score,
-          comments: postListing.num_comments
-        },
-        url: threadUrl,
-        date: new Date(postListing.created_utc * 1000).toISOString()
-      });
-      const validComments = [];
-      for (const child of commentListing) {
-        if (child.kind === "t1" && child.data.body) {
-          const comment = child.data;
-          const body = comment.body.trim();
-          if (body.length < 30 || body === "[deleted]" || body === "[removed]" || body.includes("RemindMe!") || body.endsWith("?")) {
-            continue;
-          }
-          validComments.push({
-            text: body,
-            author: comment.author,
-            engagement: {
-              score: comment.score
-            },
-            url: `https://reddit.com${comment.permalink}`,
-            date: new Date(comment.created_utc * 1000).toISOString()
-          });
-        }
-      }
-      validComments.sort((a, b) => (b.engagement.score || 0) - (a.engagement.score || 0));
-      posts.push(...validComments.slice(0, 5));
-      return posts;
-    } catch (error) {
-      if (attempts >= maxRetries) {
-        console.error(`Error fetching Reddit thread ${threadUrl} after ${maxRetries} attempts:`, error);
-        throw error;
-      }
-    }
-  }
-  return [];
-}
-async function fetchRedditThreads(threadUrls, config = {}) {
-  const { maxThreads = 20, delayMs = 2000 } = { ...DEFAULT_CONFIG, ...config };
-  const allPosts = [];
-  const limitedUrls = threadUrls.slice(0, maxThreads);
-  console.log(`[Reddit] Fetching ${limitedUrls.length} threads sequentially with jitter...`);
-  for (const url of limitedUrls) {
-    try {
-      const posts = await fetchRedditThread(url, config);
-      allPosts.push(...posts);
-      if (limitedUrls.indexOf(url) < limitedUrls.length - 1) {
-        const jitter = delayMs * 0.5 * Math.random();
-        const finalDelay = delayMs + jitter;
-        await sleep(finalDelay);
-      }
-    } catch (error) {
-      console.error(`Skipping thread due to final error: ${url}`);
-    }
-  }
-  return allPosts;
-}
-
-// src/core/services/data-prep.ts
-class DataPrepService {
-  prepare(posts) {
-    const items = posts.map((post) => {
-      return {
-        ...post
-      };
-    });
-    items.sort((a, b) => (b.engagement.score || 0) - (a.engagement.score || 0));
-    return items.slice(0, 50);
-  }
-}
-
-// src/core/services/formatting.ts
-class FormattingService {
-  formatJson(result) {
-    return JSON.stringify(result, null, 2);
-  }
-}
-
-// src/test/manual-fixed-links.ts
-async function run() {
-  const args = process.argv.slice(2);
-  const fixedUrls = args.length > 0 ? args : [
-    "https://www.reddit.com/r/iems/comments/1olzu0g/the_best_iem_builds_at_each_price_2025_ultimate/",
-    "https://www.reddit.com/r/headphones/comments/1lbcngj/new_iem_tierlist_2025/",
-    "https://www.reddit.com/r/iems/comments/1c7imln/iem_tier_list/"
-  ];
-  console.log(`\uD83E\uDDEA Starting Fixed-Link Test with ${fixedUrls.length} URLs...`);
-  try {
-    const posts = await fetchRedditThreads(fixedUrls, { maxThreads: 5 });
-    console.log(`✅ Successfully fetched ${posts.length} items (Post + Top Comments).`);
-    console.log(`
-2️⃣  Preparing raw data for AI analysis...`);
-    const dataPrep = new DataPrepService;
-    const researchItems = dataPrep.prepare(posts);
-    console.log(`
-3️⃣  Generating Output (JSON)...`);
-    const formatter = new FormattingService;
-    const mockResult = {
-      query: "Manual Test: Best IEMs 2025",
-      dateRange: { from: "2025-01-01", to: "2025-12-31" },
-      items: researchItems
-    };
-    const jsonOutput = formatter.formatJson(mockResult);
-    const fs = __require("fs");
-    const path = __require("path");
-    const outputPath = path.resolve(process.cwd(), "reddit_data_test.json");
-    fs.writeFileSync(outputPath, jsonOutput);
-    console.log(`✅ Saved test output to: ${outputPath}`);
-  } catch (error) {
-    console.error("❌ Test Failed:", error);
-  }
-}
-run();

package/templates/base/commands/test-research.md

@@ -1,9 +0,0 @@
----
-description: Test the research tool with fixed URLs.
----
-
-1. Run the fixed test script
-   > `sc-research test:fixed`
-
-2. Report status
-   > Report success if the command exits with code 0, otherwise report failure.

package/templates/base/skills/social_media_research_test.md

@@ -1,52 +0,0 @@
----
-name: social_media_research_test
-description: Legacy debug worker for fixed-link Reddit pipeline testing. Generates `reddit_data_test.json` without normal search/discovery flow and is useful for manual sanity checks.
----
-
-# Social Media Research Test Skill
-
-> Legacy/debug helper. Prefer the normal pipeline (`social_media_fetch` + analysis workers) for production research.
-
-Use this skill to sanity-check the fixed-link ingestion path with known Reddit URLs.
-
-## Workflow
-
-### 1) Run Fixed-Link Test
-
-```bash
-sc-research test:fixed [URL1] [URL2] ...
-```
-
-If no URLs are provided, built-in default Reddit links are used.
-
-### 2) Verify Output File
-
-This command writes:
-
-- `reddit_data_test.json` (in current working directory)
-
-Validate that:
-
-1. file exists
-2. JSON parses
-3. top-level `items` is a non-empty array when URLs are valid
-
-### 3) Use Output (Optional)
-
-For manual worker testing, copy/rename test output as needed (for example to `reddit_data.json`) and run one analysis worker (`social_media_rank`, `social_media_sentiment`, etc.).
-
-## Expected Output Shape
-
-`reddit_data_test.json` should contain:
-
-- `query`
-- `dateRange`
-- `items` (normalized Reddit post/comment records)
-
-`items` commonly include fields like `author`, `url`, text content, engagement, and date fields.
-
-## Critical Rules
-
-1. **Debug only**: this is not the canonical production fetch path.
-2. **No schema assumptions from legacy output**: analysis workers must still validate against `web/src/types.ts` when generating classified files.
-3. **Fail clearly**: if output file is missing or malformed, report test failure instead of continuing.