studylens 0.1.0

package/README.md

@@ -0,0 +1,107 @@
+# StudyLens
+
+AI-powered deep study assistant. Paste notes, upload files, or provide URLs — StudyLens extracts knowledge points, organizes them for browsing, and generates topic pages with AI-driven Q&A.
+
+## Features
+
+- **Multi-source ingestion** — text, PDF, DOCX, XLSX, and web URLs
+- **LLM-powered extraction** — automatically identifies knowledge points, tags, and relationships
+- **Knowledge graph** — visual force-directed graph of connected concepts
+- **Topic pages** — AI-generated study pages with version history
+- **Deep analysis** — drill down into any concept with AI-powered sub-topic expansion
+- **Smart Q&A** — ask questions about your knowledge base with context-aware answers
+- **Timeline & category views** — browse knowledge by time or subject
+- **Export** — single-page HTML export with print-optimized CSS
+- **Granularity control** — limit max knowledge points per ingestion for high-level summaries
+- **Multi-provider LLM** — supports OpenAI-compatible APIs, Ollama, and custom endpoints
+
+## Quick Start
+
+```bash
+npm run setup    # Install dependencies (server + portal)
+npm run dev      # Start server (port 3000) + dev portal (port 3001)
+```
+
+Open `http://localhost:3001` for development (hot-reload, recommended).
+
+Port 3000 serves the production build — run `npm run build` first to generate `portal/dist/`, otherwise it will only serve the API.
+
+## LLM Configuration
+
+StudyLens requires an LLM backend. On first launch the Settings panel opens automatically to guide you through setup. Three options:
+
+### Option A: Agent Maestro (recommended for GitHub Copilot users)
+
+Zero API key needed — uses your existing Copilot subscription via VS Code.
+
+1. Install the [Agent Maestro](https://marketplace.visualstudio.com/items?itemName=Joouis.agent-maestro) VS Code extension
+2. It starts a local proxy at `http://localhost:23333`
+3. In StudyLens settings, enable `agent-maestro` and test the connection
+
+### Option B: OpenAI-compatible API
+
+Works with OpenAI, Azure OpenAI, DeepSeek, or any compatible endpoint.
+
+1. In StudyLens settings, enable `openai-compatible`
+2. Set `baseUrl` (default: `https://api.openai.com/v1`), `apiKey`, and `model`
+
+### Option C: Ollama (fully local, free)
+
+Run models locally with no API key or internet required.
+
+1. Install [Ollama](https://ollama.com) and pull a model: `ollama pull llama3.2`
+2. In StudyLens settings, enable `ollama` (default URL: `http://localhost:11434`)
+
+Configuration is stored in `wiki/config/llm-config.json`. A template is at `config/llm-config.template.json`.
+
+## Project Structure
+
+```
+StudyLens/
+├── server/           # Express API server
+│   └── index.js
+├── core/             # Business logic
+│   ├── extractor.js       # Knowledge extraction prompts
+│   ├── llm-provider.js    # Multi-provider LLM client
+│   └── wiki-storage.js    # Markdown-based file storage
+├── portal/           # React frontend (Vite)
+│   └── src/
+│       ├── components/    # UI components
+│       └── lib/           # Shared utilities
+├── config/           # Configuration templates
+├── e2e/              # Playwright E2E tests
+├── tests/            # API integration tests
+├── scripts/          # Utility scripts
+└── docs/             # User & developer guides
+```
+
+## Data Storage
+
+All data is stored as Markdown files in the `wiki/` directory (gitignored by default):
+
+- `wiki/entries/` — knowledge point Markdown files with YAML frontmatter
+- `wiki/topic-pages/` — generated topic page HTML
+- `wiki/index/` — JSON indexes for fast lookup
+- `wiki/config/` — runtime configuration
+
+## Testing
+
+```bash
+npm test              # Unit tests (API + portal)
+npm run test:e2e      # Playwright E2E tests
+npm run test:api      # API tests only
+npm run test:portal   # Portal component tests only
+```
+
+## Scripts
+
+```bash
+npm run server        # Start API server only (port 3000)
+npm run portal        # Start Vite dev server only (port 3001)
+npm run dev           # Start both concurrently
+npm run setup         # Install all dependencies
+```
+
+## License
+
+MIT

package/bin/studylens.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+const path = require('path');
+const dataDir = process.env.STUDYLENS_DATA_DIR || path.join(process.cwd(), 'studylens-data');
+process.env.STUDYLENS_WIKI_DIR = process.env.STUDYLENS_WIKI_DIR || dataDir;
+require('../server/index.js');

package/config/llm-config.template.json

@@ -0,0 +1,28 @@
+{
+  "defaultProvider": "auto",
+  "providers": {
+    "agent-maestro": {
+      "enabled": false,
+      "baseUrl": "http://localhost:23333/api/anthropic",
+      "model": "claude-sonnet-4-6"
+    },
+    "openai-compatible": {
+      "enabled": false,
+      "baseUrl": "https://api.openai.com/v1",
+      "apiKey": "",
+      "model": "gpt-4o"
+    },
+    "ollama": {
+      "enabled": false,
+      "baseUrl": "http://localhost:11434",
+      "model": "llama3.2"
+    }
+  },
+  "taskRouting": {
+    "analyze": "default",
+    "questions": "default",
+    "topicPage": "default",
+    "qa": "default",
+    "expand": "default"
+  }
+}

package/config/prompts.json

@@ -0,0 +1,14 @@
+{
+  "subjects": {
+    "英语": {
+      "analyzePrompt": "You are a knowledge extraction assistant for a middle school English student. Analyze the following study notes and extract structured knowledge entries.\n\nFor each distinct knowledge point, return a JSON array of objects with:\n- \"title\": concise title in Chinese (under 20 chars), e.g. \"Unit5 比较级与最高级\"\n- \"content\": the knowledge point explained clearly in Chinese, include English examples with Chinese translations in parentheses\n- \"subject\": precise classification like \"英语-词汇与话题\", \"英语-语法\", \"英语-形容词比较级\", \"英语-写作技巧\", \"英语-动名词\" etc.\n- \"tags\": array of relevant tags including:\n  1. Grammar points: \"比较级\", \"最高级\", \"一般过去时\", \"动名词\", \"可数名词\", \"不可数名词\" etc.\n  2. Topic/Unit tags: \"自然景观\", \"运动\", \"饮食健康\" etc.\n  3. Key phrases as tags: \"be famous for\", \"imagine doing\" etc.\n  4. Skill dimensions: \"短语搭配\", \"语法规则\", \"易错点\", \"写作\" etc.\n\nImportant:\n- Each entry should focus on ONE grammar point or vocabulary cluster\n- Always include English examples with Chinese translations\n- Tag grammar-related entries with specific grammar terms",
+      "questionsPrompt": "生成的问题应该覆盖以下维度，帮助学生深入掌握英语知识点：\n1. 语法规则深挖（规则的例外情况、易错点、为什么是这样）\n2. 造句练习（要求使用特定短语或语法结构造复合句）\n3. 辨析对比（易混淆词、近义词区别、相似语法的区分）\n4. 知识迁移（把语法规则应用到新语境，如翻译句子）\n5. 综合运用（同时使用多个短语或语法点完成一个写作任务）\n\n问题要求：\n- 问题用中文提出，但涉及的英语内容保持英文\n- 鼓励学生写出完整英文句子而不是只选择答案\n- 涉及语法规则时，要求学生解释原因而不只是记忆\n- 可以引用课文中的重点短语来设计造句题\n\n返回JSON数组，每个元素: {\"question\": \"问题内容\", \"category\": \"语法/造句/辨析/迁移/综合\"}"
+    }
+  },
+  "defaultPrompts": {
+    "analyzePrompt": "You are a knowledge extraction assistant for a student. Analyze the following study notes and extract structured knowledge entries.\n\nFor each distinct knowledge point, return a JSON array of objects with:\n- \"title\": concise title (under 20 chars)\n- \"content\": the knowledge point explained clearly\n- \"subject\": precise subject classification (see rules below)\n- \"tags\": array of relevant tags — include ALL of the following dimensions:\n  1. Core concepts: key terms, names, formulas (e.g. \"科举制\", \"赵匡胤\", \"勾股定理\")\n  2. Category dimensions: assign multi-dimensional category tags based on the subject area:\n     - For history: add tags from these dimensions where applicable:\n       \"政治制度\", \"军事战争\", \"经济发展\", \"民族关系\", \"对外交流\", \"科技发明\", \"文化艺术\", \"社会生活\", \"人物\"\n     - For math: \"代数\", \"几何\", \"概率\", \"函数\", \"公式\", \"定理\", \"证明\"\n     - For physics: \"力学\", \"电磁\", \"热学\", \"光学\", \"实验\", \"公式\"\n     - For other subjects: infer appropriate dimensional tags\n  3. Connections: tags that link to related knowledge across different categories\n\nSubject classification rules:\n- For history: use specific dynasty like \"历史-隋朝\", \"历史-唐朝\", \"历史-北宋\" etc.\n- For other subjects: use patterns like \"数学-代数\", \"物理-力学\", \"化学-有机\" etc.\n- Each knowledge point must belong to exactly ONE specific category.\n\nReturn ONLY valid JSON array, no other text.",
+    "topicPrompt": "你是一个教育内容设计师。基于以下知识点和相关资料，生成一个美观的HTML专题页面。\n\n要求：\n1. 生成完整的HTML页面（含内联CSS），适合iframe嵌入\n2. 深色主题（背景 #0f1117，文字 #e0e0e0）\n3. 分章节展示：导语→背景→核心内容→影响/意义→总结\n4. 使用清晰的排版：标题、卡片、分隔线、高亮重点\n5. 中文内容，适合中学生阅读\n6. 使用你自己的知识补充完整内容，不要局限于提供的材料\n7. 页面宽度100%，无需滚动条样式\n8. 配色美观，使用渐变和阴影效果",
+    "qaPrompt": "You are an expert study assistant with deep knowledge across all subjects. A student is studying and asks you questions.\n\nIMPORTANT: Use your OWN comprehensive knowledge to answer thoroughly and accurately. The student's notes are supplementary context, not the boundary of your answer.\n\nInstructions:\n1. Answer using your full knowledge — be thorough, accurate, and educational\n2. If the student has relevant notes, reference them to build connections\n3. Use comparisons, analysis, and specific facts/data where appropriate\n4. Write in Chinese, suitable for a middle/high school student\n5. Suggest knowledge cards that capture KEY points — NEW knowledge beyond existing notes\n\nReturn a JSON object:\n{\n  \"answer\": \"Your comprehensive answer in Chinese...\",\n  \"suggestedCards\": [\n    {\n      \"title\": \"card title (under 20 chars)\",\n      \"content\": \"knowledge point explained clearly\",\n      \"subject\": \"precise subject like 历史-唐朝\",\n      \"tags\": [\"relevant\", \"tags\"]\n    }\n  ]\n}\n\nCRITICAL: The answer field must be PLAIN TEXT only — no markdown formatting.\nReturn ONLY valid JSON, no other text.",
+    "questionsPrompt": "生成的问题应该覆盖：\n1. 基本概念（是什么）\n2. 原因分析（为什么）\n3. 影响/意义（有什么影响）\n4. 比较对比（与其他知识的关联）\n5. 深入思考（评价/启示）\n\n返回JSON数组，每个元素: {\"question\": \"问题内容\", \"category\": \"概念/原因/影响/对比/思考\"}"
+  }
+}

package/core/extractor.js

@@ -0,0 +1,70 @@
+const fs = require('fs');
+const path = require('path');
+const http = require('http');
+const https = require('https');
+
+async function extractFromFile(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+  const buf = fs.readFileSync(filePath);
+
+  if (ext === '.pdf') {
+    const pdfParse = require('pdf-parse');
+    const data = await pdfParse(buf);
+    return data.text;
+  }
+
+  if (ext === '.docx') {
+    const mammoth = require('mammoth');
+    const result = await mammoth.extractRawText({ buffer: buf });
+    return result.value;
+  }
+
+  if (ext === '.xlsx' || ext === '.xls') {
+    const XLSX = require('xlsx');
+    const wb = XLSX.read(buf);
+    const texts = [];
+    for (const name of wb.SheetNames) {
+      const sheet = wb.Sheets[name];
+      texts.push(`[${name}]\n${XLSX.utils.sheet_to_csv(sheet)}`);
+    }
+    return texts.join('\n\n');
+  }
+
+  if (ext === '.txt' || ext === '.md') {
+    return buf.toString('utf-8');
+  }
+
+  throw new Error(`Unsupported file type: ${ext}`);
+}
+
+function fetchUrl(urlStr) {
+  return new Promise((resolve, reject) => {
+    const url = new URL(urlStr);
+    const mod = url.protocol === 'https:' ? https : http;
+    mod.get(url, { timeout: 30000, headers: { 'User-Agent': 'Mozilla/5.0 StudyLens/1.0' } }, (res) => {
+      if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+        return fetchUrl(res.headers.location).then(resolve, reject);
+      }
+      if (res.statusCode >= 400) return reject(new Error(`HTTP ${res.statusCode}`));
+      const chunks = [];
+      res.on('data', c => chunks.push(c));
+      res.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    }).on('error', reject);
+  });
+}
+
+async function extractFromUrl(urlStr) {
+  const html = await fetchUrl(urlStr);
+  const cheerio = require('cheerio');
+  const $ = cheerio.load(html);
+
+  $('script, style, nav, footer, header, iframe, noscript').remove();
+
+  const article = $('article').length ? $('article') : $('main').length ? $('main') : $('body');
+  const text = article.text().replace(/\s+/g, ' ').trim();
+
+  if (text.length < 50) throw new Error('Could not extract meaningful text from URL');
+  return text.slice(0, 15000);
+}
+
+module.exports = { extractFromFile, extractFromUrl };