relay-os 0.1.0

package/backend/parsers/copilot.js

@@ -0,0 +1,379 @@
+const fs = require('fs');
+const path = require('path');
+const { readVscdbJson, vscodeInternalTimeToIso } = require('../lib/vscdb');
+const { getWorkspacePathFromStorageDir } = require('../lib/vscodeWorkspace');
+const {
+  scanHistoryForWorkspace,
+  getCodeHistoryRoots,
+  enrichEditsWithHistory,
+  mergeCodeEditSources,
+} = require('../lib/vscodeHistory');
+const { parseCopilotEditingSessions, listEditingSessionDirs } = require('./copilotEditingSessions');
+
+const SOURCE = 'GitHub Copilot';
+
+const extractText = (value) => {
+  if (Array.isArray(value)) {
+    return value
+      .map(part => {
+        if (typeof part === 'string') return part;
+        if (part && typeof part === 'object') {
+          if (part.type === 'text' && typeof part.text === 'string') return part.text;
+          if (typeof part.content === 'string') return part.content;
+        }
+        return '';
+      })
+      .join(' ')
+      .trim();
+  }
+
+  if (typeof value === 'string') return value.trim();
+  if (value && typeof value === 'object') {
+    if (typeof value.text === 'string') return value.text.trim();
+    if (typeof value.content === 'string') return value.content.trim();
+  }
+
+  return String(value || '').trim();
+};
+
+function extractUserRequest(renderedUserMessage) {
+  for (const part of renderedUserMessage || []) {
+    if (typeof part?.text !== 'string') continue;
+    const match = part.text.match(/<userRequest>\s*([\s\S]*?)\s*<\/userRequest>/i);
+    if (match) return match[1].trim();
+  }
+  return null;
+}
+
+function foldChatSessionPatches(lines) {
+  let state = null;
+
+  for (const line of lines) {
+    let step;
+    try {
+      step = JSON.parse(line);
+    } catch (_) {
+      continue;
+    }
+
+    if (step.kind === 0) {
+      state = step.v;
+      continue;
+    }
+
+    if (!state || step.kind !== 1 || !Array.isArray(step.k)) continue;
+
+    let current = state;
+    for (let i = 0; i < step.k.length - 1; i++) {
+      const key = step.k[i];
+      if (current[key] === undefined) {
+        current[key] = typeof step.k[i + 1] === 'number' ? [] : {};
+      }
+      current = current[key];
+    }
+    current[step.k[step.k.length - 1]] = step.v;
+  }
+
+  return state;
+}
+
+function parseCopilotEventsJsonl(transcriptPath) {
+  const content = fs.readFileSync(transcriptPath, 'utf-8');
+  const lines = content.trim().split('\n').filter(Boolean);
+  const events = [];
+  let sessionStartTs = null;
+
+  for (const line of lines) {
+    try {
+      const step = JSON.parse(line);
+
+      if (step.type === 'session.start') {
+        sessionStartTs = step.timestamp || step.data?.startTime || null;
+        continue;
+      }
+
+      if (step.type === 'user.message' && step.data?.content) {
+        const text = extractText(step.data.content);
+        if (text) {
+          events.push({
+            ts: step.timestamp,
+            kind: 'message',
+            role: 'user',
+            content: text,
+            source: SOURCE,
+          });
+        }
+      } else if (step.type === 'assistant.message' && step.data?.content) {
+        const text = extractText(step.data.content);
+        if (text) {
+          events.push({
+            ts: step.timestamp,
+            kind: 'message',
+            role: 'assistant',
+            content: text,
+            source: SOURCE,
+          });
+        }
+      }
+    } catch (_) { }
+  }
+
+  return { events, sessionStartTs };
+}
+
+function parseCopilotChatSession(chatSessionPath) {
+  const content = fs.readFileSync(chatSessionPath, 'utf-8');
+  const lines = content.trim().split('\n').filter(Boolean);
+  if (!lines.length) return { events: [], sessionId: null, sessionStartTs: null };
+
+  let firstStep;
+  try {
+    firstStep = JSON.parse(lines[0]);
+  } catch (_) {
+    return { events: [], sessionId: null, sessionStartTs: null };
+  }
+
+  if (firstStep.kind !== 0) {
+    return parseCopilotEventsJsonl(chatSessionPath);
+  }
+
+  const state = foldChatSessionPatches(lines);
+  if (!state) return { events: [], sessionId: null, sessionStartTs: null };
+
+  const sessionId = state.sessionId || path.basename(chatSessionPath, '.jsonl');
+  const sessionStartTs = state.creationDate
+    ? new Date(state.creationDate).toISOString()
+    : null;
+
+  const events = [];
+  const requests = Array.isArray(state.requests) ? state.requests : [];
+
+  for (let i = 0; i < requests.length; i++) {
+    const request = requests[i];
+    const userText =
+      extractUserRequest(request?.result?.metadata?.renderedUserMessage) ||
+      (i === 0 && state.customTitle ? String(state.customTitle).trim() : null);
+
+    if (userText) {
+      events.push({
+        ts: sessionStartTs,
+        kind: 'message',
+        role: 'user',
+        content: userText,
+        source: SOURCE,
+        sessionId,
+        requestIndex: i,
+      });
+    }
+  }
+
+  return { events, sessionId, sessionStartTs };
+}
+
+function findSiblingTranscript(chatSessionPath, sessionId) {
+  const workspaceDir = path.dirname(path.dirname(chatSessionPath));
+  const transcriptPath = path.join(
+    workspaceDir,
+    'GitHub.copilot-chat',
+    'transcripts',
+    `${sessionId}.jsonl`
+  );
+  return fs.existsSync(transcriptPath) ? transcriptPath : null;
+}
+
+function mergeCopilotEvents(userEvents, transcriptEvents, sessionStartTs) {
+  const merged = userEvents.map(event => ({
+    ...event,
+    ts: sessionStartTs || event.ts,
+  }));
+
+  for (const event of transcriptEvents) {
+    merged.push(event);
+  }
+
+  merged.sort((a, b) => {
+    const ta = Date.parse(a.ts || '') || 0;
+    const tb = Date.parse(b.ts || '') || 0;
+    if (ta !== tb) return ta - tb;
+    if (a.role === 'user' && b.role !== 'user') return -1;
+    if (a.role !== 'user' && b.role === 'user') return 1;
+    return 0;
+  });
+
+  if (sessionStartTs && merged.length) {
+    const hasUser = merged.some(e => e.role === 'user');
+    if (!hasUser) {
+      merged.unshift({
+        ts: sessionStartTs,
+        kind: 'message',
+        role: 'user',
+        content: '(session started)',
+        source: SOURCE,
+      });
+    }
+  }
+
+  return merged;
+}
+
+function parseSingleChatSession(chatSessionPath) {
+  const { events: userEvents, sessionId, sessionStartTs } = parseCopilotChatSession(chatSessionPath);
+  const transcriptFile = sessionId ? findSiblingTranscript(chatSessionPath, sessionId) : null;
+
+  if (transcriptFile) {
+    const { events: transcriptEvents, sessionStartTs: transcriptStartTs } =
+      parseCopilotEventsJsonl(transcriptFile);
+    return mergeCopilotEvents(
+      userEvents,
+      transcriptEvents,
+      transcriptStartTs || sessionStartTs
+    );
+  }
+
+  return userEvents;
+}
+
+function getWorkspaceStorageDir(inputPath) {
+  if (!inputPath) return null;
+  if (!fs.existsSync(inputPath)) return null;
+
+  const stat = fs.statSync(inputPath);
+  if (stat.isDirectory()) return inputPath;
+
+  let dir = path.dirname(inputPath);
+  while (dir && path.basename(dir) !== 'workspaceStorage') {
+    const base = path.basename(dir);
+    if (base === 'chatSessions' || base === 'transcripts' || base === 'GitHub.copilot-chat') {
+      return path.dirname(path.dirname(dir));
+    }
+    if (fs.existsSync(path.join(dir, 'workspace.json'))) return dir;
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+
+  return null;
+}
+
+function listChatSessionFiles(workspaceStorageDir) {
+  const dir = path.join(workspaceStorageDir, 'chatSessions');
+  if (!fs.existsSync(dir)) return [];
+
+  return fs.readdirSync(dir, { withFileTypes: true })
+    .filter(entry => entry.isFile() && entry.name.endsWith('.jsonl'))
+    .map(entry => path.join(dir, entry.name));
+}
+
+function assignEditTimestamps(codeEdits, sessionIndex) {
+  return codeEdits.map(event => {
+    const timing = sessionIndex?.entries?.[event.sessionId]?.timing;
+    if (!timing) {
+      return { ...event, ts: event.ts || null };
+    }
+
+    const start = Number(timing.lastRequestStarted || timing.created) || 0;
+    const end = Number(timing.lastRequestEnded || timing.lastRequestStarted || start) || start;
+    const span = Math.max(end - start, 1);
+    const epoch = Number(event.epoch) || 0;
+    const maxEpoch = 10;
+    const ts = vscodeInternalTimeToIso(start + Math.floor((span * epoch) / maxEpoch));
+
+    return { ...event, ts: ts || event.ts };
+  }).filter(event => event.ts);
+}
+
+function parseCopilotWorkspace(workspaceStorageDir, options = {}) {
+  const sessionIndex = readVscdbJson(
+    path.join(workspaceStorageDir, 'state.vscdb'),
+    'chat.ChatSessionStore.index'
+  );
+
+  const chatEvents = [];
+  for (const chatSessionPath of listChatSessionFiles(workspaceStorageDir)) {
+    chatEvents.push(...parseSingleChatSession(chatSessionPath));
+  }
+
+  const workspacePath =
+    options.workspacePath || getWorkspacePathFromStorageDir(workspaceStorageDir);
+
+  const sessionEdits = assignEditTimestamps(
+    parseCopilotEditingSessions(workspaceStorageDir, sessionIndex),
+    sessionIndex
+  );
+
+  const checkpoints = sessionEdits.filter(event => event.kind === 'checkpoint');
+  const chatEditingEdits = sessionEdits.filter(event => event.kind === 'code_edit');
+
+  const historyEdits = workspacePath
+    ? scanHistoryForWorkspace(workspacePath, { roots: getCodeHistoryRoots() })
+    : [];
+
+  const enrichedSessionEdits = enrichEditsWithHistory(chatEditingEdits, historyEdits);
+  const codeEdits = mergeCodeEditSources(historyEdits, enrichedSessionEdits);
+
+  const events = [...chatEvents, ...codeEdits, ...checkpoints];
+  events.sort((a, b) => {
+    const ta = Date.parse(a.ts || '') || 0;
+    const tb = Date.parse(b.ts || '') || 0;
+    if (ta !== tb) return ta - tb;
+    const kindOrder = { message: 0, code_edit: 1, artifact: 2, checkpoint: 3 };
+    return (kindOrder[a.kind] ?? 9) - (kindOrder[b.kind] ?? 9);
+  });
+
+  return {
+    events,
+    workspaceStorageDir,
+    workspacePath,
+    sessionIndex,
+    sources: {
+      chatSessions: listChatSessionFiles(workspaceStorageDir),
+      editingSessions: listEditingSessionDirs(workspaceStorageDir).length,
+      vscdb: fs.existsSync(path.join(workspaceStorageDir, 'state.vscdb')),
+      localHistory: historyEdits.length,
+    },
+  };
+}
+
+// Reads Copilot chat + code edits from workspaceStorage (chatSessions, transcripts,
+// chatEditingSessions, state.vscdb index, Code/User/History snapshots), or legacy paths.
+function parseCopilot(transcriptPath, options = {}) {
+  if (!transcriptPath || !fs.existsSync(transcriptPath)) return [];
+
+  const workspaceDir = getWorkspaceStorageDir(transcriptPath);
+  if (workspaceDir && fs.statSync(transcriptPath).isDirectory()) {
+    return parseCopilotWorkspace(workspaceDir, options).events;
+  }
+
+  if (workspaceDir && path.basename(path.dirname(transcriptPath)) === 'chatSessions') {
+    return parseCopilotWorkspace(workspaceDir, options).events;
+  }
+
+  let firstLine;
+  try {
+    firstLine = fs.readFileSync(transcriptPath, 'utf-8').split('\n')[0];
+  } catch (_) {
+    return [];
+  }
+
+  let firstStep;
+  try {
+    firstStep = JSON.parse(firstLine);
+  } catch (_) {
+    return [];
+  }
+
+  if (firstStep.kind === 0) {
+    return parseSingleChatSession(transcriptPath);
+  }
+
+  return parseCopilotEventsJsonl(transcriptPath).events;
+}
+
+module.exports = {
+  parseCopilot,
+  parseCopilotWorkspace,
+  parseCopilotChatSession,
+  parseCopilotEventsJsonl,
+  foldChatSessionPatches,
+  getWorkspaceStorageDir,
+};

package/backend/parsers/copilotEditingSessions.js

@@ -0,0 +1,118 @@
+const fs = require('fs');
+const path = require('path');
+
+const SOURCE = 'GitHub Copilot';
+
+function uriToPath(uri) {
+  if (!uri) return null;
+  if (typeof uri === 'string') {
+    if (uri.startsWith('file:///')) {
+      try {
+        return decodeURIComponent(uri.replace('file:///', '')).replace(/\//g, path.sep);
+      } catch (_) {
+        return null;
+      }
+    }
+    return uri;
+  }
+  if (uri.fsPath) return uri.fsPath;
+  if (uri.external?.startsWith('file:///')) return uriToPath(uri.external);
+  return null;
+}
+
+function listEditingSessionDirs(workspaceStorageDir) {
+  const root = path.join(workspaceStorageDir, 'chatEditingSessions');
+  if (!fs.existsSync(root)) return [];
+
+  return fs.readdirSync(root, { withFileTypes: true })
+    .filter(entry => entry.isDirectory())
+    .map(entry => path.join(root, entry.name));
+}
+
+function parseEditingSessionDir(sessionDir, sessionTiming) {
+  const statePath = path.join(sessionDir, 'state.json');
+  if (!fs.existsSync(statePath)) return [];
+
+  let state;
+  try {
+    state = JSON.parse(fs.readFileSync(statePath, 'utf-8'));
+  } catch (_) {
+    return [];
+  }
+
+  const sessionId = path.basename(sessionDir);
+  const events = [];
+  const timeline = state.timeline || {};
+  const checkpoints = timeline.checkpoints || [];
+  const operations = timeline.operations || [];
+  const epochToCheckpoint = new Map(checkpoints.map(cp => [cp.epoch, cp]));
+
+  const baseTs = sessionTiming?.lastRequestStarted || sessionTiming?.created || null;
+
+  for (const op of operations) {
+    const filePath = uriToPath(op.uri);
+    if (!filePath || op.type !== 'textEdit') continue;
+
+    const editPreview = (op.edits || [])
+      .map(e => String(e.text || '').trim())
+      .filter(Boolean)
+      .join(' ')
+      .slice(0, 160);
+
+    const checkpoint = epochToCheckpoint.get(op.epoch);
+    events.push({
+      ts: baseTs,
+      kind: 'code_edit',
+      role: 'assistant',
+      source: SOURCE,
+      sessionId,
+      file: path.basename(filePath),
+      path: filePath,
+      action: 'textEdit',
+      editSource: 'chatEditingSession',
+      epoch: op.epoch,
+      requestId: op.requestId || checkpoint?.requestId || null,
+      summary: editPreview || checkpoint?.label || 'Copilot chat edit',
+      checkpointId: checkpoint?.checkpointId || null,
+    });
+  }
+
+  for (const cp of checkpoints) {
+    if (cp.epoch === 0) continue;
+    const fileUri = state.initialFileContents?.[0]?.[0];
+    const filePath = uriToPath(fileUri);
+    events.push({
+      ts: baseTs,
+      kind: 'checkpoint',
+      role: 'assistant',
+      source: SOURCE,
+      sessionId,
+      file: filePath ? path.basename(filePath) : null,
+      path: filePath,
+      epoch: cp.epoch,
+      requestId: cp.requestId || null,
+      summary: cp.label || cp.description || 'Copilot edit checkpoint',
+      checkpointId: cp.checkpointId,
+    });
+  }
+
+  return events;
+}
+
+function parseCopilotEditingSessions(workspaceStorageDir, sessionIndex) {
+  const dirs = listEditingSessionDirs(workspaceStorageDir);
+  const events = [];
+
+  for (const dir of dirs) {
+    const sessionId = path.basename(dir);
+    const timing = sessionIndex?.entries?.[sessionId]?.timing || null;
+    events.push(...parseEditingSessionDir(dir, timing));
+  }
+
+  return events;
+}
+
+module.exports = {
+  parseCopilotEditingSessions,
+  listEditingSessionDirs,
+};

package/backend/parsers/cursor.js

@@ -0,0 +1,215 @@
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { normalizeTs, makeUnifiedDiff, isRedactedContent } = require('../lib/timeline');
+const { scanHistoryForWorkspace, getCursorHistoryRoots } = require('../lib/vscodeHistory');
+
+const SOURCE = 'Cursor';
+const EDIT_TOOLS = new Set(['Write', 'StrReplace', 'ApplyPatch', 'EditNotebook', 'search_replace']);
+
+const HOME = os.homedir();
+
+function workspaceToCursorSlug(workspacePath) {
+  const norm = String(workspacePath || '')
+    .replace(/\\/g, '/')
+    .replace(/\/$/, '');
+
+  const driveMatch = norm.match(/^([a-zA-Z]):\/?(.*)$/);
+  if (driveMatch) {
+    const tail = driveMatch[2].replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
+    return `${driveMatch[1].toLowerCase()}-${tail}`;
+  }
+
+  let slug = norm.replace(/[^a-zA-Z0-9]/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
+  if (slug.charAt(1) === '-') slug = slug.charAt(0).toLowerCase() + slug.slice(1);
+  return slug;
+}
+
+function discoverCursorTranscriptDir(workspacePath) {
+  const projectsRoot = path.join(HOME, '.cursor', 'projects');
+  const slug = workspaceToCursorSlug(workspacePath);
+  const direct = path.join(projectsRoot, slug, 'agent-transcripts');
+  if (fs.existsSync(direct)) return direct;
+
+  if (!fs.existsSync(projectsRoot)) return null;
+
+  const target = workspacePath.toLowerCase().replace(/\\/g, '/').replace(/\/$/, '');
+  for (const entry of fs.readdirSync(projectsRoot, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const candidate = path.join(projectsRoot, entry.name, 'agent-transcripts');
+    if (!fs.existsSync(candidate)) continue;
+
+    for (const file of listCursorTranscriptFiles(candidate).slice(0, 3)) {
+      try {
+        const sample = fs.readFileSync(file, 'utf-8').slice(0, 8000).toLowerCase();
+        if (sample.includes(target)) return candidate;
+      } catch (_) { }
+    }
+  }
+
+  return null;
+}
+
+function listCursorTranscriptFiles(transcriptDir) {
+  if (!transcriptDir || !fs.existsSync(transcriptDir)) return [];
+
+  const files = [];
+  for (const entry of fs.readdirSync(transcriptDir, { withFileTypes: true })) {
+    if (entry.isDirectory()) {
+      const jsonl = path.join(transcriptDir, entry.name, `${entry.name}.jsonl`);
+      if (fs.existsSync(jsonl)) files.push(jsonl);
+    } else if (entry.name.endsWith('.jsonl')) {
+      files.push(path.join(transcriptDir, entry.name));
+    }
+  }
+
+  return files.sort((a, b) => {
+    try { return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs; } catch { return 0; }
+  });
+}
+
+function extractUserText(content) {
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter(part => part?.type === 'text' && typeof part.text === 'string')
+    .map(part => {
+      const match = part.text.match(/<user_query>\s*([\s\S]*?)\s*<\/user_query>/i);
+      return match ? match[1].trim() : part.text.trim();
+    })
+    .filter(Boolean)
+    .join('\n');
+}
+
+function extractAssistantText(content) {
+  if (!Array.isArray(content)) return '';
+  return content
+    .filter(part => part?.type === 'text' && typeof part.text === 'string')
+    .map(part => part.text.trim())
+    .filter(Boolean)
+    .join('\n');
+}
+
+function toolToCodeEdit(tool, ts) {
+  const name = tool.name;
+  if (!EDIT_TOOLS.has(name)) return null;
+
+  const input = tool.input || {};
+  const filePath = input.path || input.file_path || input.target_notebook || null;
+  if (!filePath) return null;
+
+  let diff = null;
+  let summary = `${name} ${path.basename(filePath)}`;
+
+  if (name === 'StrReplace' && input.old_string != null && input.new_string != null) {
+    diff = makeUnifiedDiff(input.old_string, input.new_string, path.basename(filePath));
+    summary = `Edited ${path.basename(filePath)}`;
+  } else if (name === 'Write' && input.contents != null) {
+    diff = makeUnifiedDiff('', input.contents, path.basename(filePath));
+    summary = `Wrote ${path.basename(filePath)}`;
+  }
+
+  return {
+    ts,
+    kind: 'code_edit',
+    role: 'assistant',
+    source: SOURCE,
+    file: path.basename(filePath),
+    path: filePath,
+    action: name.toLowerCase(),
+    summary,
+    diff,
+  };
+}
+
+function parseCursorTranscriptFile(transcriptPath, baseTs) {
+  const content = fs.readFileSync(transcriptPath, 'utf-8');
+  const lines = content.trim().split('\n').filter(Boolean);
+  const events = [];
+  const fileMtime = fs.statSync(transcriptPath).mtimeMs;
+  const startTs = baseTs || normalizeTs(fileMtime);
+
+  lines.forEach((line, index) => {
+    try {
+      const step = JSON.parse(line);
+      const ts = startTs
+        ? normalizeTs(Date.parse(startTs) + index * 1000)
+        : normalizeTs(fileMtime + index * 1000);
+
+      if (step.role === 'user') {
+        const text = extractUserText(step.message?.content);
+        if (text && !isRedactedContent(text)) {
+          events.push({
+            ts,
+            kind: 'message',
+            role: 'user',
+            content: text,
+            source: SOURCE,
+            transcriptPath,
+            lineIndex: index,
+          });
+        }
+      } else if (step.role === 'assistant') {
+        const text = extractAssistantText(step.message?.content);
+        if (text && !isRedactedContent(text)) {
+          events.push({
+            ts,
+            kind: 'message',
+            role: 'assistant',
+            content: text,
+            source: SOURCE,
+            transcriptPath,
+            lineIndex: index,
+          });
+        }
+
+        const tools = Array.isArray(step.message?.content)
+          ? step.message.content.filter(part => part?.type === 'tool_use')
+          : [];
+
+        for (const tool of tools) {
+          const edit = toolToCodeEdit(tool, ts);
+          if (edit) events.push(edit);
+        }
+      }
+    } catch (_) { }
+  });
+
+  return events;
+}
+
+function parseCursor(inputPath, options = {}) {
+  const workspacePath = options.workspacePath;
+  let transcriptFiles = [];
+
+  if (inputPath && fs.existsSync(inputPath)) {
+    if (fs.statSync(inputPath).isDirectory()) {
+      transcriptFiles = listCursorTranscriptFiles(inputPath);
+    } else {
+      transcriptFiles = [inputPath];
+    }
+  } else if (workspacePath) {
+    const dir = discoverCursorTranscriptDir(workspacePath);
+    if (dir) transcriptFiles = listCursorTranscriptFiles(dir);
+  }
+
+  const events = [];
+  for (const file of transcriptFiles) {
+    events.push(...parseCursorTranscriptFile(file));
+  }
+
+  if (workspacePath) {
+    events.push(...scanHistoryForWorkspace(workspacePath, {
+      roots: getCursorHistoryRoots(),
+    }));
+  }
+
+  events.sort((a, b) => Date.parse(a.ts || '') - Date.parse(b.ts || ''));
+  return { events, transcriptFiles };
+}
+
+module.exports = {
+  parseCursor,
+  discoverCursorTranscriptDir,
+  listCursorTranscriptFiles,
+  workspaceToCursorSlug,
+};