tokrepo 2.0.0 → 3.1.0

package/bin/tokrepo.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const crypto = require('crypto');
 const https = require('https');
 const http = require('http');
 const readline = require('readline');
@@ -21,8 +22,11 @@ const C = {
 
 const CONFIG_DIR = path.join(require('os').homedir(), '.tokrepo');
 const CONFIG_FILE = path.join(CONFIG_DIR, 'config.json');
+const SYNC_STATE_FILE = path.join(CONFIG_DIR, 'sync-state.json');
 const PROJECT_CONFIG = '.tokrepo.json';
 const DEFAULT_API = 'https://api.tokrepo.com';
+const CLI_VERSION = '3.1.0';
+const VERSION_CHECK_FILE = path.join(require('os').homedir(), '.tokrepo', '.version-check');
 
 // ─── Helpers ───
 
@@ -42,9 +46,73 @@ function readConfig() {
 
 function writeConfig(config) {
   if (!fs.existsSync(CONFIG_DIR)) {
-    fs.mkdirSync(CONFIG_DIR, { recursive: true });
+    fs.mkdirSync(CONFIG_DIR, { recursive: true, mode: 0o700 });
   }
-  fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2));
+  fs.writeFileSync(CONFIG_FILE, JSON.stringify(config, null, 2), { mode: 0o600 });
+}
+
+// Check npm registry for newer version (non-blocking, max once per day)
+async function checkForUpdate() {
+  try {
+    // Only check once per 24 hours
+    if (fs.existsSync(VERSION_CHECK_FILE)) {
+      const stat = fs.statSync(VERSION_CHECK_FILE);
+      const hoursSinceCheck = (Date.now() - stat.mtimeMs) / 3600000;
+      if (hoursSinceCheck < 24) return;
+    }
+    // Touch the file to mark check time
+    if (!fs.existsSync(path.dirname(VERSION_CHECK_FILE))) {
+      fs.mkdirSync(path.dirname(VERSION_CHECK_FILE), { recursive: true, mode: 0o700 });
+    }
+    fs.writeFileSync(VERSION_CHECK_FILE, '', { mode: 0o600 });
+
+    const data = await new Promise((resolve, reject) => {
+      const req = https.get('https://registry.npmjs.org/tokrepo/latest', {
+        headers: { 'Accept': 'application/json' },
+        timeout: 3000,
+      }, (res) => {
+        let body = '';
+        res.on('data', (chunk) => { body += chunk; });
+        res.on('end', () => {
+          try { resolve(JSON.parse(body)); } catch { reject(); }
+        });
+      });
+      req.on('error', reject);
+      req.on('timeout', () => { req.destroy(); reject(); });
+    });
+
+    const latest = data.version;
+    if (latest && latest !== CLI_VERSION) {
+      const cmp = compareVersions(latest, CLI_VERSION);
+      if (cmp > 0) {
+        log('');
+        log(`${C.yellow}!${C.reset} Update available: ${C.dim}${CLI_VERSION}${C.reset} → ${C.green}${latest}${C.reset}`);
+        log(`  Run: ${C.cyan}npm install -g tokrepo${C.reset}`);
+        log('');
+      }
+    }
+  } catch {
+    // Silent fail — update check is best-effort
+  }
+}
+
+function compareVersions(a, b) {
+  const pa = a.split('.').map(Number);
+  const pb = b.split('.').map(Number);
+  for (let i = 0; i < 3; i++) {
+    if ((pa[i] || 0) > (pb[i] || 0)) return 1;
+    if ((pa[i] || 0) < (pb[i] || 0)) return -1;
+  }
+  return 0;
+}
+
+// Sync state: maps "dirPath:title" → { uuid, hash, lastSync }
+function readSyncState() {
+  try { return JSON.parse(fs.readFileSync(SYNC_STATE_FILE, 'utf8')); } catch { return {}; }
+}
+function writeSyncState(state) {
+  if (!fs.existsSync(CONFIG_DIR)) fs.mkdirSync(CONFIG_DIR, { recursive: true, mode: 0o700 });
+  fs.writeFileSync(SYNC_STATE_FILE, JSON.stringify(state, null, 2), { mode: 0o600 });
 }
 
 function readProjectConfig(baseDir = process.cwd()) {
@@ -74,6 +142,10 @@ function apiRequest(method, urlPath, body, token, apiBase) {
   return new Promise((resolve, reject) => {
     const base = apiBase || DEFAULT_API;
     const url = new URL(urlPath, base);
+    // Force HTTPS to prevent token transmission over plain HTTP
+    if (url.protocol === 'http:' && !url.hostname.match(/^(localhost|127\.0\.0\.1)$/)) {
+      url.protocol = 'https:';
+    }
     const isHttps = url.protocol === 'https:';
     const mod = isHttps ? https : http;
 
@@ -144,7 +216,8 @@ function detectFileType(filename) {
 
 // Guess tag from file type
 function guessTag(fileType) {
-  const map = { skill: 'Skills', prompt: 'Prompts', script: 'Scripts', config: 'Configs' };
+  // Use lowercase singular names to match backend tag slugs
+  const map = { skill: 'skill', prompt: 'prompt', script: 'script', config: 'config', mcp: 'mcp' };
   return map[fileType] || null;
 }
 
@@ -253,6 +326,11 @@ function collectFiles(paths, baseDir) {
 
   for (const p of paths) {
     const resolved = path.resolve(baseDir, p);
+    // Prevent path traversal — resolved must be within baseDir
+    if (!resolved.startsWith(path.resolve(baseDir) + path.sep) && resolved !== path.resolve(baseDir)) {
+      warn(`Skipped (outside project): ${p}`);
+      continue;
+    }
     if (!fs.existsSync(resolved)) {
       warn(`Not found: ${p}`);
       continue;
@@ -674,6 +752,9 @@ Examples:
       }
       case 'mcp':
       case 'mcp configs': {
+        // Security warning for MCP configs
+        warn('MCP server config detected. Review the configuration carefully before adding to your project.');
+        warn('MCP servers can execute arbitrary code. Only install from trusted sources.');
         // Save as mcp config, hint about manual merge
         if (!fileName.endsWith('.json')) fileName = fileName.replace(/\.md$/, '.json');
         break;
@@ -704,6 +785,8 @@ Examples:
       }
     }
 
+    // Sanitize fileName to prevent path traversal from API response
+    fileName = path.basename(fileName);
     const destPath = path.join(destDir, fileName);
 
     // Don't overwrite without warning
@@ -848,6 +931,351 @@ async function cmdTags() {
   }
 }
 
+// ─── Sync: scan directory, diff with remote, upsert changes ───
+
+function computeHash(files) {
+  const h = crypto.createHash('sha256');
+  for (const f of files) {
+    h.update(f.name);
+    h.update('\0');
+    h.update(f.content);
+    h.update('\0');
+  }
+  return h.digest('hex');
+}
+
+function scanDirectory(dirPath) {
+  const assets = [];
+  const SKIP_DIRS = new Set(['node_modules', '.git', '__pycache__', '.venv', 'dist', 'build']);
+  const SKIP_FILES = new Set(['.DS_Store', 'Thumbs.db', 'package-lock.json', 'yarn.lock']);
+
+  // Each subdirectory = one asset; loose files = one asset per file
+  let entries;
+  try { entries = fs.readdirSync(dirPath, { withFileTypes: true }); } catch { return assets; }
+
+  const looseFiles = [];
+
+  for (const entry of entries) {
+    if (entry.name.startsWith('.') || SKIP_DIRS.has(entry.name) || SKIP_FILES.has(entry.name)) continue;
+
+    const fullPath = path.join(dirPath, entry.name);
+
+    if (entry.isDirectory()) {
+      // Subdirectory = one asset (e.g., ~/.claude/skills/my-skill/)
+      const files = collectAssetFiles(fullPath);
+      if (files.length === 0) continue;
+
+      const title = guessAssetTitle(files, entry.name);
+      const hash = computeHash(files);
+      const detectedTags = new Set();
+      for (const f of files) {
+        const ft = detectFileType(f.name);
+        const tag = guessTag(ft);
+        if (tag) detectedTags.add(tag);
+      }
+
+      assets.push({ title, files, hash, tags: Array.from(detectedTags), sourcePath: fullPath });
+    } else if (entry.isFile()) {
+      // Loose file
+      const ext = path.extname(entry.name).toLowerCase();
+      const validExts = ['.md', '.sh', '.py', '.js', '.mjs', '.ts', '.json', '.yaml', '.yml', '.toml', '.prompt'];
+      if (validExts.includes(ext) || entry.name === '.cursorrules' || entry.name === '.windsurfrules') {
+        looseFiles.push({ path: fullPath, name: entry.name });
+      }
+    }
+  }
+
+  // Each loose file = one asset
+  for (const f of looseFiles) {
+    let content;
+    try { content = fs.readFileSync(f.path, 'utf8'); } catch { continue; }
+    if (!content.trim()) continue;
+
+    const files = [{ name: f.name, content, type: detectFileType(f.name) }];
+    const title = guessAssetTitle(files, path.basename(f.name, path.extname(f.name)));
+    const hash = computeHash(files);
+    const ft = detectFileType(f.name);
+    const tag = guessTag(ft);
+
+    assets.push({ title, files, hash, tags: tag ? [tag] : [], sourcePath: f.path });
+  }
+
+  return assets;
+}
+
+function collectAssetFiles(dirPath) {
+  const files = [];
+  const SKIP = new Set(['.DS_Store', 'node_modules', '.git', '__pycache__']);
+
+  function walk(dir, relBase) {
+    let entries;
+    try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries) {
+      if (entry.name.startsWith('.') || SKIP.has(entry.name)) continue;
+      const fullPath = path.join(dir, entry.name);
+      const relPath = relBase ? `${relBase}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) {
+        walk(fullPath, relPath);
+      } else if (entry.isFile()) {
+        const ext = path.extname(entry.name).toLowerCase();
+        const validExts = ['.md', '.sh', '.py', '.js', '.mjs', '.ts', '.json', '.yaml', '.yml', '.toml', '.prompt', '.rb', '.go', '.rs'];
+        if (validExts.includes(ext) || entry.name === '.cursorrules') {
+          let content;
+          try { content = fs.readFileSync(fullPath, 'utf8'); } catch { continue; }
+          if (!content.trim()) continue;
+          files.push({ name: relPath, content, type: detectFileType(relPath) });
+        }
+      }
+    }
+  }
+
+  walk(dirPath, '');
+  return files;
+}
+
+function guessAssetTitle(files, fallbackName) {
+  // Try to find a heading in the first .md file
+  for (const f of files) {
+    if (f.name.toLowerCase().endsWith('.md')) {
+      const match = f.content.match(/^#\s+(.+)$/m);
+      if (match) return match[1].trim();
+    }
+  }
+  // Clean up fallback name
+  return fallbackName
+    .replace(/[-_]/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+async function cmdSync() {
+  const args = parseArgs(process.argv);
+  const config = readConfig();
+  if (!config || !config.token) error(`Not logged in. Run: ${C.cyan}tokrepo login${C.reset}`);
+
+  const targetDir = args.positional[0]
+    ? path.resolve(args.positional[0])
+    : path.join(require('os').homedir(), '.claude', 'skills');
+
+  if (!fs.existsSync(targetDir)) {
+    error(`Directory not found: ${targetDir}`);
+  }
+
+  const visibility = args.flags.public ? 1 : 0; // default private for sync
+
+  log(`\n${C.bold}tokrepo sync${C.reset}\n`);
+  info(`Scanning ${targetDir}...`);
+
+  const assets = scanDirectory(targetDir);
+  if (assets.length === 0) {
+    info('No assets found in directory.');
+    return;
+  }
+
+  log(`  Found ${C.bold}${assets.length}${C.reset} assets\n`);
+
+  // Load local sync state for fast local-only diff
+  const syncState = readSyncState();
+
+  // Classify each asset: check local state first, then remote diff for unknowns
+  const needsRemoteCheck = [];
+  const localStatus = {};
+
+  for (const asset of assets) {
+    const key = asset.sourcePath;
+    const cached = syncState[key];
+    if (cached && cached.hash === asset.hash) {
+      // Local hash matches last sync — unchanged
+      localStatus[asset.title] = { status: 'unchanged', uuid: cached.uuid };
+    } else if (cached && cached.hash !== asset.hash) {
+      // Local hash differs from last sync — updated
+      localStatus[asset.title] = { status: 'updated', uuid: cached.uuid };
+    } else {
+      // Not in local state — check remote
+      needsRemoteCheck.push(asset);
+    }
+  }
+
+  // Remote diff for assets not in local state
+  if (needsRemoteCheck.length > 0) {
+    info('Comparing with remote...');
+    const diffPayload = needsRemoteCheck.map(a => ({ title: a.title, content_hash: a.hash }));
+    try {
+      const data = await apiRequest('POST', '/api/v1/tokenboard/push/diff', { assets: diffPayload }, config.token, config.api);
+      for (const r of data.results) {
+        localStatus[r.title] = { status: r.status, uuid: r.remote_uuid || '' };
+      }
+    } catch (e) {
+      warn(`Diff API: ${e.message} — treating unknowns as new`);
+      for (const a of needsRemoteCheck) {
+        localStatus[a.title] = { status: 'new', uuid: '' };
+      }
+    }
+  }
+
+  // Show status
+  let newCount = 0, updatedCount = 0, unchangedCount = 0;
+
+  for (const asset of assets) {
+    const st = localStatus[asset.title] || { status: 'new' };
+    if (st.status === 'new') {
+      log(`  ${C.green}+ new${C.reset}       ${asset.title} ${C.dim}(${asset.files.length} files)${C.reset}`);
+      newCount++;
+    } else if (st.status === 'updated') {
+      log(`  ${C.yellow}~ updated${C.reset}   ${asset.title}`);
+      updatedCount++;
+    } else {
+      log(`  ${C.dim}= unchanged ${asset.title}${C.reset}`);
+      unchangedCount++;
+    }
+  }
+
+  log('');
+  log(`  ${C.green}${newCount} new${C.reset}  ${C.yellow}${updatedCount} updated${C.reset}  ${C.dim}${unchangedCount} unchanged${C.reset}`);
+
+  if (newCount === 0 && updatedCount === 0) {
+    log('');
+    success('Everything is up to date!');
+    return;
+  }
+
+  log('');
+
+  // Confirm unless -y
+  if (!args.flags.yes) {
+    const confirm = await ask(`Push ${newCount + updatedCount} assets? (y/N):`);
+    if (confirm.toLowerCase() !== 'y') { log('Aborted.'); return; }
+  }
+
+  // Upsert each changed asset + save state
+  let successCount = 0;
+  let failCount = 0;
+
+  for (const asset of assets) {
+    const st = localStatus[asset.title] || { status: 'new' };
+    if (st.status === 'unchanged') continue;
+
+    const totalChars = asset.files.reduce((sum, f) => sum + f.content.length, 0);
+
+    try {
+      const data = await apiRequest('POST', '/api/v1/tokenboard/push/upsert', {
+        title: asset.title,
+        files: asset.files,
+        tags: asset.tags,
+        token_cost: String(Math.round(totalChars / 4)),
+        visibility,
+      }, config.token, config.api);
+
+      const action = data.action === 'created' ? C.green + '+ created' : C.yellow + '~ updated';
+      log(`  ${action}${C.reset}  ${asset.title}  ${C.dim}${data.url}${C.reset}`);
+
+      // Save to local sync state
+      syncState[asset.sourcePath] = {
+        uuid: data.uuid,
+        hash: asset.hash,
+        title: asset.title,
+        url: data.url,
+        lastSync: new Date().toISOString(),
+      };
+      successCount++;
+    } catch (e) {
+      log(`  ${C.red}✗ failed${C.reset}   ${asset.title}: ${e.message}`);
+      failCount++;
+    }
+  }
+
+  // Persist sync state
+  writeSyncState(syncState);
+
+  log('');
+  if (failCount === 0) {
+    success(`Synced ${successCount} assets!`);
+  } else {
+    warn(`${successCount} synced, ${failCount} failed`);
+  }
+  log('');
+}
+
+async function cmdStatus() {
+  const args = parseArgs(process.argv);
+  const config = readConfig();
+  if (!config || !config.token) error(`Not logged in. Run: ${C.cyan}tokrepo login${C.reset}`);
+
+  const targetDir = args.positional[0]
+    ? path.resolve(args.positional[0])
+    : path.join(require('os').homedir(), '.claude', 'skills');
+
+  if (!fs.existsSync(targetDir)) {
+    error(`Directory not found: ${targetDir}`);
+  }
+
+  log(`\n${C.bold}tokrepo status${C.reset}\n`);
+  info(`Scanning ${targetDir}...`);
+
+  const assets = scanDirectory(targetDir);
+  if (assets.length === 0) {
+    info('No assets found in directory.');
+    return;
+  }
+
+  // Check local sync state first, remote for unknowns
+  const syncState = readSyncState();
+  const localStatus = {};
+  const needsRemoteCheck = [];
+
+  for (const asset of assets) {
+    const cached = syncState[asset.sourcePath];
+    if (cached && cached.hash === asset.hash) {
+      localStatus[asset.title] = { status: 'unchanged', uuid: cached.uuid };
+    } else if (cached && cached.hash !== asset.hash) {
+      localStatus[asset.title] = { status: 'updated', uuid: cached.uuid };
+    } else {
+      needsRemoteCheck.push(asset);
+    }
+  }
+
+  if (needsRemoteCheck.length > 0) {
+    info('Comparing with remote...');
+    const diffPayload = needsRemoteCheck.map(a => ({ title: a.title, content_hash: a.hash }));
+    try {
+      const data = await apiRequest('POST', '/api/v1/tokenboard/push/diff', { assets: diffPayload }, config.token, config.api);
+      for (const r of data.results) {
+        localStatus[r.title] = { status: r.status, uuid: r.remote_uuid || '' };
+      }
+    } catch (e) {
+      error(`Diff API error: ${e.message}`);
+    }
+  }
+
+  log('');
+  let newCount = 0, updatedCount = 0, unchangedCount = 0;
+
+  for (const asset of assets) {
+    const st = localStatus[asset.title] || { status: 'new', uuid: '' };
+    const status = st.status;
+    const uuid = st.uuid || '';
+    const uuidShort = uuid ? ` ${C.dim}(${uuid.substring(0, 8)})${C.reset}` : '';
+
+    if (status === 'new') {
+      log(`  ${C.green}+ new${C.reset}       ${asset.title} ${C.dim}(${asset.files.length} files)${C.reset}`);
+      newCount++;
+    } else if (status === 'updated') {
+      log(`  ${C.yellow}~ modified${C.reset}  ${asset.title}${uuidShort}`);
+      updatedCount++;
+    } else {
+      log(`  ${C.dim}  unchanged ${asset.title}${uuidShort}${C.reset}`);
+      unchangedCount++;
+    }
+  }
+
+  log('');
+  log(`  ${C.bold}${assets.length}${C.reset} local assets: ${C.green}${newCount} new${C.reset}  ${C.yellow}${updatedCount} modified${C.reset}  ${C.dim}${unchangedCount} synced${C.reset}`);
+
+  if (newCount > 0 || updatedCount > 0) {
+    log(`\n  Run ${C.cyan}tokrepo sync ${args.positional[0] || ''}${C.reset} to push changes`);
+  }
+  log('');
+}
+
 function showHelp() {
   log(`
 ${C.bold}tokrepo${C.reset} — AI assets for humans and agents. Like GitHub, for AI experience.
@@ -865,10 +1293,12 @@ ${C.bold}DISCOVER & INSTALL${C.reset}
   ${C.cyan}install${C.reset} <name|uuid> Smart install (auto-detects type & placement)
   ${C.cyan}pull${C.reset} <url|uuid>     Download raw asset files
 
-${C.bold}PUBLISH${C.reset}
-  ${C.cyan}push${C.reset} [files...]     Push files/directory to TokRepo
+${C.bold}PUBLISH & SYNC${C.reset}
+  ${C.cyan}push${C.reset} [files...]     Push files/directory (creates new asset)
+  ${C.cyan}sync${C.reset} [dir]          Sync directory to TokRepo (smart upsert)
+  ${C.cyan}status${C.reset} [dir]        Show local vs remote diff
   ${C.cyan}init${C.reset}                Create .tokrepo.json project config
-  ${C.cyan}update${C.reset} <uuid> [f]   Update existing asset
+  ${C.cyan}update${C.reset} <uuid> [f]   Update existing asset by UUID
   ${C.cyan}delete${C.reset} <uuid>       Delete an asset
 
 ${C.bold}ACCOUNT${C.reset}
@@ -892,6 +1322,16 @@ ${C.bold}INSTALL BEHAVIOR${C.reset}
   MCP      → current dir        (.json)
   Prompts  → current dir        (.md)
 
+${C.bold}SYNC (the killer feature)${C.reset}
+  ${C.cyan}tokrepo sync ~/.claude/skills/${C.reset}          # Sync all skills (default: private)
+  ${C.cyan}tokrepo sync ~/.claude/skills/ --public${C.reset} # Sync as public assets
+  ${C.cyan}tokrepo sync . -y${C.reset}                       # Sync current dir, skip confirm
+  ${C.cyan}tokrepo status${C.reset}                          # Show what would change
+
+  Sync scans a directory, detects new/modified assets, and pushes only
+  what changed. Each subdirectory becomes one asset. Loose files become
+  individual assets. Like ${C.bold}git push${C.reset} for your AI assets.
+
 ${C.bold}EXAMPLES${C.reset}
   tokrepo search "mcp server"                 # Find MCP configs
   tokrepo install ca000374-f5d8-...           # Install by UUID
@@ -919,6 +1359,8 @@ async function main() {
     case 'login': await cmdLogin(); break;
     case 'init': await cmdInit(); break;
     case 'push': await cmdPush(); break;
+    case 'sync': await cmdSync(); break;
+    case 'status': case 'st': await cmdStatus(); break;
     case 'pull': await cmdPull(); break;
     case 'search': case 'find': await cmdSearch(); break;
     case 'install': case 'i': await cmdInstall(); break;
@@ -927,11 +1369,16 @@ async function main() {
     case 'delete': await cmdDelete(); break;
     case 'tags': await cmdTags(); break;
     case 'whoami': await cmdWhoami(); break;
+    case '--version': case '-v': case 'version':
+      log(`tokrepo ${CLI_VERSION}`); break;
     case 'help': case '--help': case '-h': case undefined:
       showHelp(); break;
     default:
       error(`Unknown command: ${command}. Run: tokrepo help`);
   }
+
+  // Non-blocking update check after command completes
+  checkForUpdate();
 }
 
 main().catch((e) => { error(e.message); });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "tokrepo",
-  "version": "2.0.0",
-  "description": "AI assets for humans and agents — search, install, push. Like GitHub, for AI experience.",
+  "version": "3.1.0",
+  "description": "AI assets for humans and agents — sync, search, install, push. Like git push for your AI skills.",
   "bin": {
     "tokrepo": "bin/tokrepo.js"
   },