xtctx 0.3.2 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +267 -122
- package/LICENSE +21 -21
- package/README.md +154 -121
- package/dist/src/cli/disconnect.d.ts +8 -0
- package/dist/src/cli/disconnect.d.ts.map +1 -0
- package/dist/src/cli/disconnect.js +42 -0
- package/dist/src/cli/disconnect.js.map +1 -0
- package/dist/src/cli/hook.d.ts +7 -0
- package/dist/src/cli/hook.d.ts.map +1 -0
- package/dist/src/cli/hook.js +21 -0
- package/dist/src/cli/hook.js.map +1 -0
- package/dist/src/cli/index.d.ts.map +1 -1
- package/dist/src/cli/index.js +71 -33
- package/dist/src/cli/index.js.map +1 -1
- package/dist/src/cli/setup.d.ts +8 -0
- package/dist/src/cli/setup.d.ts.map +1 -0
- package/dist/src/cli/setup.js +77 -0
- package/dist/src/cli/setup.js.map +1 -0
- package/dist/src/cli/status.d.ts +7 -0
- package/dist/src/cli/status.d.ts.map +1 -0
- package/dist/src/cli/status.js +84 -0
- package/dist/src/cli/status.js.map +1 -0
- package/dist/src/config/disconnect.d.ts +32 -0
- package/dist/src/config/disconnect.d.ts.map +1 -0
- package/dist/src/config/disconnect.js +304 -0
- package/dist/src/config/disconnect.js.map +1 -0
- package/dist/src/config/mcp-config.d.ts +91 -0
- package/dist/src/config/mcp-config.d.ts.map +1 -0
- package/dist/src/config/mcp-config.js +462 -0
- package/dist/src/config/mcp-config.js.map +1 -0
- package/dist/src/config/setup.d.ts +37 -0
- package/dist/src/config/setup.d.ts.map +1 -0
- package/dist/src/config/setup.js +338 -0
- package/dist/src/config/setup.js.map +1 -0
- package/dist/src/config/skills.d.ts +73 -0
- package/dist/src/config/skills.d.ts.map +1 -0
- package/dist/src/config/skills.js +427 -0
- package/dist/src/config/skills.js.map +1 -0
- package/dist/src/handoff/embeddings.d.ts +15 -0
- package/dist/src/handoff/embeddings.d.ts.map +1 -0
- package/dist/src/handoff/embeddings.js +63 -0
- package/dist/src/handoff/embeddings.js.map +1 -0
- package/dist/src/handoff/sqlite-index.d.ts +43 -0
- package/dist/src/handoff/sqlite-index.d.ts.map +1 -0
- package/dist/src/handoff/sqlite-index.js +718 -0
- package/dist/src/handoff/sqlite-index.js.map +1 -0
- package/dist/src/handoff/types.d.ts +57 -0
- package/dist/src/handoff/types.d.ts.map +1 -0
- package/dist/src/handoff/types.js +2 -0
- package/dist/src/handoff/types.js.map +1 -0
- package/dist/src/handoff/vector.d.ts +4 -0
- package/dist/src/handoff/vector.d.ts.map +1 -0
- package/dist/src/handoff/vector.js +31 -0
- package/dist/src/handoff/vector.js.map +1 -0
- package/dist/src/mcp/server.d.ts +2 -19
- package/dist/src/mcp/server.d.ts.map +1 -1
- package/dist/src/mcp/server.js +75 -199
- package/dist/src/mcp/server.js.map +1 -1
- package/dist/src/mcp/tools/continuity.d.ts +3 -0
- package/dist/src/mcp/tools/continuity.d.ts.map +1 -0
- package/dist/src/mcp/tools/continuity.js +30 -0
- package/dist/src/mcp/tools/continuity.js.map +1 -0
- package/dist/src/mcp/tools/manifest.d.ts +40 -0
- package/dist/src/mcp/tools/manifest.d.ts.map +1 -0
- package/dist/src/mcp/tools/manifest.js +96 -0
- package/dist/src/mcp/tools/manifest.js.map +1 -0
- package/dist/src/mcp/tools/sessions.d.ts +11 -32
- package/dist/src/mcp/tools/sessions.d.ts.map +1 -1
- package/dist/src/mcp/tools/sessions.js +47 -13
- package/dist/src/mcp/tools/sessions.js.map +1 -1
- package/dist/src/runtime/services.d.ts +11 -38
- package/dist/src/runtime/services.d.ts.map +1 -1
- package/dist/src/runtime/services.js +43 -202
- package/dist/src/runtime/services.js.map +1 -1
- package/dist/src/scrapers/antigravity.d.ts +41 -0
- package/dist/src/scrapers/antigravity.d.ts.map +1 -0
- package/dist/src/scrapers/antigravity.js +858 -0
- package/dist/src/scrapers/antigravity.js.map +1 -0
- package/dist/src/scrapers/base.d.ts +53 -1
- package/dist/src/scrapers/base.d.ts.map +1 -1
- package/dist/src/scrapers/base.js +89 -3
- package/dist/src/scrapers/base.js.map +1 -1
- package/dist/src/scrapers/claude-code.d.ts +20 -6
- package/dist/src/scrapers/claude-code.d.ts.map +1 -1
- package/dist/src/scrapers/claude-code.js +144 -26
- package/dist/src/scrapers/claude-code.js.map +1 -1
- package/dist/src/scrapers/codex.d.ts +36 -6
- package/dist/src/scrapers/codex.d.ts.map +1 -1
- package/dist/src/scrapers/codex.js +256 -46
- package/dist/src/scrapers/codex.js.map +1 -1
- package/dist/src/scrapers/copilot-cli.d.ts +33 -0
- package/dist/src/scrapers/copilot-cli.d.ts.map +1 -0
- package/dist/src/scrapers/copilot-cli.js +257 -0
- package/dist/src/scrapers/copilot-cli.js.map +1 -0
- package/dist/src/scrapers/copilot.d.ts +41 -8
- package/dist/src/scrapers/copilot.d.ts.map +1 -1
- package/dist/src/scrapers/copilot.js +256 -118
- package/dist/src/scrapers/copilot.js.map +1 -1
- package/dist/src/scrapers/cursor.d.ts +24 -9
- package/dist/src/scrapers/cursor.d.ts.map +1 -1
- package/dist/src/scrapers/cursor.js +247 -151
- package/dist/src/scrapers/cursor.js.map +1 -1
- package/dist/src/scrapers/index.d.ts +18 -0
- package/dist/src/scrapers/index.d.ts.map +1 -0
- package/dist/src/scrapers/index.js +17 -0
- package/dist/src/scrapers/index.js.map +1 -0
- package/dist/src/scrapers/opencode.d.ts +36 -0
- package/dist/src/scrapers/opencode.d.ts.map +1 -0
- package/dist/src/scrapers/opencode.js +248 -0
- package/dist/src/scrapers/opencode.js.map +1 -0
- package/dist/src/scrapers/registry.d.ts +1 -0
- package/dist/src/scrapers/registry.d.ts.map +1 -1
- package/dist/src/scrapers/registry.js +3 -0
- package/dist/src/scrapers/registry.js.map +1 -1
- package/dist/src/tools/sources.d.ts +28 -0
- package/dist/src/tools/sources.d.ts.map +1 -0
- package/dist/src/tools/sources.js +149 -0
- package/dist/src/tools/sources.js.map +1 -0
- package/dist/src/types/config.d.ts +24 -37
- package/dist/src/types/config.d.ts.map +1 -1
- package/dist/src/types/index.d.ts +0 -2
- package/dist/src/types/index.d.ts.map +1 -1
- package/dist/src/types/index.js +0 -2
- package/dist/src/types/index.js.map +1 -1
- package/dist/src/types/scraper.d.ts +34 -5
- package/dist/src/types/scraper.d.ts.map +1 -1
- package/dist/src/utils/errors.d.ts +2 -0
- package/dist/src/utils/errors.d.ts.map +1 -0
- package/dist/src/utils/errors.js +7 -0
- package/dist/src/utils/errors.js.map +1 -0
- package/dist/src/utils/package-info.d.ts +11 -0
- package/dist/src/utils/package-info.d.ts.map +1 -0
- package/dist/src/utils/package-info.js +30 -0
- package/dist/src/utils/package-info.js.map +1 -0
- package/dist/src/utils/project-scope.d.ts +3 -0
- package/dist/src/utils/project-scope.d.ts.map +1 -0
- package/dist/src/utils/project-scope.js +45 -0
- package/dist/src/utils/project-scope.js.map +1 -0
- package/package.json +70 -66
- package/dist/src/api/routes/config.d.ts +0 -4
- package/dist/src/api/routes/config.d.ts.map +0 -1
- package/dist/src/api/routes/config.js +0 -61
- package/dist/src/api/routes/config.js.map +0 -1
- package/dist/src/api/routes/knowledge.d.ts +0 -4
- package/dist/src/api/routes/knowledge.d.ts.map +0 -1
- package/dist/src/api/routes/knowledge.js +0 -40
- package/dist/src/api/routes/knowledge.js.map +0 -1
- package/dist/src/api/routes/search.d.ts +0 -7
- package/dist/src/api/routes/search.d.ts.map +0 -1
- package/dist/src/api/routes/search.js +0 -108
- package/dist/src/api/routes/search.js.map +0 -1
- package/dist/src/api/routes/sources.d.ts +0 -12
- package/dist/src/api/routes/sources.d.ts.map +0 -1
- package/dist/src/api/routes/sources.js +0 -70
- package/dist/src/api/routes/sources.js.map +0 -1
- package/dist/src/api/server.d.ts +0 -26
- package/dist/src/api/server.d.ts.map +0 -1
- package/dist/src/api/server.js +0 -312
- package/dist/src/api/server.js.map +0 -1
- package/dist/src/cli/ingest.d.ts +0 -6
- package/dist/src/cli/ingest.d.ts.map +0 -1
- package/dist/src/cli/ingest.js +0 -11
- package/dist/src/cli/ingest.js.map +0 -1
- package/dist/src/cli/init.d.ts +0 -6
- package/dist/src/cli/init.d.ts.map +0 -1
- package/dist/src/cli/init.js +0 -105
- package/dist/src/cli/init.js.map +0 -1
- package/dist/src/cli/serve.d.ts +0 -7
- package/dist/src/cli/serve.d.ts.map +0 -1
- package/dist/src/cli/serve.js +0 -100
- package/dist/src/cli/serve.js.map +0 -1
- package/dist/src/cli/sync.d.ts +0 -5
- package/dist/src/cli/sync.d.ts.map +0 -1
- package/dist/src/cli/sync.js +0 -13
- package/dist/src/cli/sync.js.map +0 -1
- package/dist/src/compaction/pipeline.d.ts +0 -19
- package/dist/src/compaction/pipeline.d.ts.map +0 -1
- package/dist/src/compaction/pipeline.js +0 -37
- package/dist/src/compaction/pipeline.js.map +0 -1
- package/dist/src/compaction/providers/cli.d.ts +0 -13
- package/dist/src/compaction/providers/cli.d.ts.map +0 -1
- package/dist/src/compaction/providers/cli.js +0 -60
- package/dist/src/compaction/providers/cli.js.map +0 -1
- package/dist/src/compaction/rule-based.d.ts +0 -8
- package/dist/src/compaction/rule-based.d.ts.map +0 -1
- package/dist/src/compaction/rule-based.js +0 -120
- package/dist/src/compaction/rule-based.js.map +0 -1
- package/dist/src/config/loader.d.ts +0 -12
- package/dist/src/config/loader.d.ts.map +0 -1
- package/dist/src/config/loader.js +0 -76
- package/dist/src/config/loader.js.map +0 -1
- package/dist/src/config/mappings/claude-code.d.ts +0 -3
- package/dist/src/config/mappings/claude-code.d.ts.map +0 -1
- package/dist/src/config/mappings/claude-code.js +0 -31
- package/dist/src/config/mappings/claude-code.js.map +0 -1
- package/dist/src/config/mappings/codex.d.ts +0 -3
- package/dist/src/config/mappings/codex.d.ts.map +0 -1
- package/dist/src/config/mappings/codex.js +0 -31
- package/dist/src/config/mappings/codex.js.map +0 -1
- package/dist/src/config/mappings/copilot.d.ts +0 -3
- package/dist/src/config/mappings/copilot.d.ts.map +0 -1
- package/dist/src/config/mappings/copilot.js +0 -33
- package/dist/src/config/mappings/copilot.js.map +0 -1
- package/dist/src/config/mappings/cursor.d.ts +0 -3
- package/dist/src/config/mappings/cursor.d.ts.map +0 -1
- package/dist/src/config/mappings/cursor.js +0 -29
- package/dist/src/config/mappings/cursor.js.map +0 -1
- package/dist/src/config/sync.d.ts +0 -13
- package/dist/src/config/sync.d.ts.map +0 -1
- package/dist/src/config/sync.js +0 -96
- package/dist/src/config/sync.js.map +0 -1
- package/dist/src/ingestion/coordinator.d.ts +0 -37
- package/dist/src/ingestion/coordinator.d.ts.map +0 -1
- package/dist/src/ingestion/coordinator.js +0 -127
- package/dist/src/ingestion/coordinator.js.map +0 -1
- package/dist/src/ingestion/daemon.d.ts +0 -17
- package/dist/src/ingestion/daemon.d.ts.map +0 -1
- package/dist/src/ingestion/daemon.js +0 -35
- package/dist/src/ingestion/daemon.js.map +0 -1
- package/dist/src/ingestion/watcher.d.ts +0 -19
- package/dist/src/ingestion/watcher.d.ts.map +0 -1
- package/dist/src/ingestion/watcher.js +0 -70
- package/dist/src/ingestion/watcher.js.map +0 -1
- package/dist/src/knowledge/autotag.d.ts +0 -3
- package/dist/src/knowledge/autotag.d.ts.map +0 -1
- package/dist/src/knowledge/autotag.js +0 -24
- package/dist/src/knowledge/autotag.js.map +0 -1
- package/dist/src/knowledge/dedup.d.ts +0 -7
- package/dist/src/knowledge/dedup.d.ts.map +0 -1
- package/dist/src/knowledge/dedup.js +0 -15
- package/dist/src/knowledge/dedup.js.map +0 -1
- package/dist/src/knowledge/repository.d.ts +0 -12
- package/dist/src/knowledge/repository.d.ts.map +0 -1
- package/dist/src/knowledge/repository.js +0 -74
- package/dist/src/knowledge/repository.js.map +0 -1
- package/dist/src/mcp/tools/config.d.ts +0 -36
- package/dist/src/mcp/tools/config.d.ts.map +0 -1
- package/dist/src/mcp/tools/config.js +0 -54
- package/dist/src/mcp/tools/config.js.map +0 -1
- package/dist/src/mcp/tools/knowledge.d.ts +0 -17
- package/dist/src/mcp/tools/knowledge.d.ts.map +0 -1
- package/dist/src/mcp/tools/knowledge.js +0 -46
- package/dist/src/mcp/tools/knowledge.js.map +0 -1
- package/dist/src/mcp/tools/search.d.ts +0 -22
- package/dist/src/mcp/tools/search.d.ts.map +0 -1
- package/dist/src/mcp/tools/search.js +0 -30
- package/dist/src/mcp/tools/search.js.map +0 -1
- package/dist/src/mcp/tools/write.d.ts +0 -38
- package/dist/src/mcp/tools/write.d.ts.map +0 -1
- package/dist/src/mcp/tools/write.js +0 -97
- package/dist/src/mcp/tools/write.js.map +0 -1
- package/dist/src/runtime/ingestion.d.ts +0 -17
- package/dist/src/runtime/ingestion.d.ts.map +0 -1
- package/dist/src/runtime/ingestion.js +0 -106
- package/dist/src/runtime/ingestion.js.map +0 -1
- package/dist/src/scrapers/gemini.d.ts +0 -17
- package/dist/src/scrapers/gemini.d.ts.map +0 -1
- package/dist/src/scrapers/gemini.js +0 -245
- package/dist/src/scrapers/gemini.js.map +0 -1
- package/dist/src/store/embeddings.d.ts +0 -9
- package/dist/src/store/embeddings.d.ts.map +0 -1
- package/dist/src/store/embeddings.js +0 -28
- package/dist/src/store/embeddings.js.map +0 -1
- package/dist/src/store/lance.d.ts +0 -25
- package/dist/src/store/lance.d.ts.map +0 -1
- package/dist/src/store/lance.js +0 -74
- package/dist/src/store/lance.js.map +0 -1
- package/dist/src/store/search.d.ts +0 -18
- package/dist/src/store/search.d.ts.map +0 -1
- package/dist/src/store/search.js +0 -55
- package/dist/src/store/search.js.map +0 -1
- package/dist/src/types/compaction.d.ts +0 -29
- package/dist/src/types/compaction.d.ts.map +0 -1
- package/dist/src/types/compaction.js +0 -2
- package/dist/src/types/compaction.js.map +0 -1
- package/dist/src/types/context.d.ts +0 -33
- package/dist/src/types/context.d.ts.map +0 -1
- package/dist/src/types/context.js +0 -18
- package/dist/src/types/context.js.map +0 -1
- package/dist/src/utils/retry.d.ts +0 -11
- package/dist/src/utils/retry.d.ts.map +0 -1
- package/dist/src/utils/retry.js +0 -46
- package/dist/src/utils/retry.js.map +0 -1
- package/dist/src/utils/shutdown.d.ts +0 -25
- package/dist/src/utils/shutdown.d.ts.map +0 -1
- package/dist/src/utils/shutdown.js +0 -56
- package/dist/src/utils/shutdown.js.map +0 -1
- package/dist/web/apple-touch-icon.png +0 -0
- package/dist/web/assets/index-Nuadr5ox.css +0 -1
- package/dist/web/assets/index-aA0u8eXA.js +0 -3504
- package/dist/web/assets/primeicons-C6QP2o4f.woff2 +0 -0
- package/dist/web/assets/primeicons-DMOk5skT.eot +0 -0
- package/dist/web/assets/primeicons-Dr5RGzOO.svg +0 -345
- package/dist/web/assets/primeicons-MpK4pl85.ttf +0 -0
- package/dist/web/assets/primeicons-WjwUDZjB.woff +0 -0
- package/dist/web/favicon.ico +0 -0
- package/dist/web/favicon.svg +0 -21
- package/dist/web/index.html +0 -21
- package/dist/web/site.webmanifest +0 -19
- package/dist/web/vite.svg +0 -1
|
@@ -1,64 +1,52 @@
|
|
|
1
|
-
import { stat } from "node:fs/promises";
|
|
2
|
-
import {
|
|
1
|
+
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
import { fileURLToPath } from "node:url";
|
|
3
4
|
import Database from "better-sqlite3";
|
|
4
5
|
import { glob } from "glob";
|
|
5
|
-
import { estimateTokens,
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
6
|
+
import { AbstractScraper, estimateTokens, toDate } from "./base.js";
|
|
7
|
+
import { pathMatchesProject } from "../utils/project-scope.js";
|
|
8
|
+
// Bubble type constants from Cursor's internal format.
|
|
9
|
+
const BUBBLE_TYPE_USER = 1;
|
|
10
|
+
const BUBBLE_TYPE_ASSISTANT = 2;
|
|
11
|
+
const SCRAPER_NAME = "cursor";
|
|
12
|
+
/**
|
|
13
|
+
* Shapes the cursor scraper tolerates silently without logging. All other
|
|
14
|
+
* shape surprises warn; missing required tables throw.
|
|
15
|
+
*/
|
|
16
|
+
export const ACCEPTED_DEGRADATIONS = {
|
|
17
|
+
/** Store path missing — Cursor not installed on this machine. */
|
|
18
|
+
missingStorePath: "cursor workspaceStorage path absent",
|
|
19
|
+
/** Workspace has no composerData yet — empty workspace. */
|
|
20
|
+
emptyWorkspace: "workspace has no composer.composerData row",
|
|
21
|
+
/** A composer whose bubble row is missing — bubble pruned by Cursor. */
|
|
22
|
+
prunedBubble: "bubble referenced by composer but missing from globalStorage",
|
|
23
|
+
/** Empty bubble text (tool-call only, etc.). */
|
|
24
|
+
emptyBubbleText: "bubble has no user-visible text",
|
|
25
|
+
/** Forward-compat unknown keys alongside known composer fields. */
|
|
26
|
+
unknownFieldsAlongside: "extra keys alongside known composer schema",
|
|
13
27
|
};
|
|
14
|
-
|
|
15
|
-
`
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
`,
|
|
27
|
-
`
|
|
28
|
-
SELECT
|
|
29
|
-
id AS rowId,
|
|
30
|
-
conversation_id AS sessionId,
|
|
31
|
-
author AS role,
|
|
32
|
-
text AS content,
|
|
33
|
-
timestamp,
|
|
34
|
-
model,
|
|
35
|
-
mode AS composerMode
|
|
36
|
-
FROM chat_messages
|
|
37
|
-
ORDER BY timestamp ASC
|
|
38
|
-
`,
|
|
39
|
-
`
|
|
40
|
-
SELECT
|
|
41
|
-
id AS rowId,
|
|
42
|
-
sessionId,
|
|
43
|
-
role,
|
|
44
|
-
content,
|
|
45
|
-
timestamp,
|
|
46
|
-
model,
|
|
47
|
-
composerMode
|
|
48
|
-
FROM cursor_messages
|
|
49
|
-
ORDER BY timestamp ASC
|
|
50
|
-
`,
|
|
51
|
-
];
|
|
52
|
-
export class CursorScraper {
|
|
28
|
+
function warnDrift(sourcePath, surprise, recordsAffected) {
|
|
29
|
+
console.warn(`[${SCRAPER_NAME}] schema-drift surprise at ${sourcePath}: ${surprise} ` +
|
|
30
|
+
`(records affected: ${recordsAffected})`);
|
|
31
|
+
}
|
|
32
|
+
function describeType(value) {
|
|
33
|
+
if (value === null)
|
|
34
|
+
return "null";
|
|
35
|
+
if (Array.isArray(value))
|
|
36
|
+
return "array";
|
|
37
|
+
return typeof value;
|
|
38
|
+
}
|
|
39
|
+
export class CursorScraper extends AbstractScraper {
|
|
53
40
|
cursorStorePath;
|
|
41
|
+
projectRoot;
|
|
54
42
|
tool = "cursor";
|
|
55
|
-
|
|
56
|
-
|
|
43
|
+
constructor(cursorStorePath, stateDir, projectRoot) {
|
|
44
|
+
super(stateDir);
|
|
57
45
|
this.cursorStorePath = cursorStorePath;
|
|
58
|
-
this.
|
|
46
|
+
this.projectRoot = projectRoot;
|
|
59
47
|
}
|
|
60
48
|
async detect() {
|
|
61
|
-
const paths = await this.
|
|
49
|
+
const paths = await this.resolveWorkspaceDatabasePaths();
|
|
62
50
|
return paths.length > 0;
|
|
63
51
|
}
|
|
64
52
|
getStorePaths() {
|
|
@@ -72,80 +60,164 @@ export class CursorScraper {
|
|
|
72
60
|
async *fullSync() {
|
|
73
61
|
yield* this.readAllMessages(new Date(0));
|
|
74
62
|
}
|
|
75
|
-
parseRaw(raw) {
|
|
76
|
-
const value = raw;
|
|
77
|
-
const timestamp = coerceDate(value.timestamp);
|
|
78
|
-
const sessionId = toNonEmptyString(value.sessionId) ?? "unknown";
|
|
79
|
-
const content = toNonEmptyString(value.content) ?? "";
|
|
80
|
-
const role = normalizeRole(toNonEmptyString(value.role));
|
|
81
|
-
const model = toNonEmptyString(value.model) ?? "unknown";
|
|
82
|
-
const composerMode = normalizeComposerMode(toNonEmptyString(value.composerMode));
|
|
83
|
-
return {
|
|
84
|
-
tool: "cursor",
|
|
85
|
-
sessionId,
|
|
86
|
-
timestamp,
|
|
87
|
-
role,
|
|
88
|
-
content,
|
|
89
|
-
metadata: {
|
|
90
|
-
messageIndex: toMessageIndex(value.messageIndex),
|
|
91
|
-
tokenEstimate: estimateTokens(content),
|
|
92
|
-
referencedFiles: [],
|
|
93
|
-
model,
|
|
94
|
-
composerMode,
|
|
95
|
-
},
|
|
96
|
-
};
|
|
97
|
-
}
|
|
98
|
-
async getLastScrapedPosition() {
|
|
99
|
-
return this.stateManager.load(this.tool);
|
|
100
|
-
}
|
|
101
|
-
async saveScrapedPosition(state) {
|
|
102
|
-
await this.stateManager.save(this.tool, state);
|
|
103
|
-
}
|
|
104
63
|
async *readAllMessages(since) {
|
|
105
|
-
const
|
|
106
|
-
for (const
|
|
107
|
-
|
|
64
|
+
const workspacePaths = await this.resolveWorkspaceDatabasePaths();
|
|
65
|
+
for (const wsPath of workspacePaths) {
|
|
66
|
+
const composerRefs = this.readWorkspaceComposers(wsPath);
|
|
67
|
+
if (composerRefs.length === 0) {
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const globalPath = deriveGlobalStoragePath(wsPath);
|
|
71
|
+
if (!globalPath) {
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
let globalDb = null;
|
|
108
75
|
try {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const messageIndexBySession = new Map();
|
|
112
|
-
for (const row of rows) {
|
|
113
|
-
const timestamp = coerceDate(row.timestamp);
|
|
114
|
-
if (timestamp <= since) {
|
|
115
|
-
continue;
|
|
116
|
-
}
|
|
117
|
-
const sessionId = row.sessionId ?? inferSessionId(dbPath);
|
|
118
|
-
const nextIndex = messageIndexBySession.get(sessionId) ?? 0;
|
|
119
|
-
messageIndexBySession.set(sessionId, nextIndex + 1);
|
|
120
|
-
yield this.parseRaw({
|
|
121
|
-
...row,
|
|
122
|
-
sessionId,
|
|
123
|
-
timestamp,
|
|
124
|
-
messageIndex: nextIndex,
|
|
125
|
-
});
|
|
126
|
-
}
|
|
76
|
+
globalDb = new Database(globalPath, { readonly: true, fileMustExist: true });
|
|
77
|
+
yield* this.readComposerMessages(globalDb, composerRefs, since, wsPath);
|
|
127
78
|
}
|
|
128
|
-
catch {
|
|
129
|
-
//
|
|
79
|
+
catch (err) {
|
|
80
|
+
// Global storage unreadable — treat as schema drift and warn.
|
|
81
|
+
// The cursorDiskKV table is required; if it's gone, something changed.
|
|
82
|
+
warnDrift(globalPath, `globalStorage unreadable: ${err.message}`, composerRefs.length);
|
|
130
83
|
}
|
|
131
84
|
finally {
|
|
132
|
-
|
|
85
|
+
globalDb?.close();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
readWorkspaceComposers(wsDbPath) {
|
|
90
|
+
let db = null;
|
|
91
|
+
try {
|
|
92
|
+
db = new Database(wsDbPath, { readonly: true, fileMustExist: true });
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
// File missing / unopenable — treat as absent workspace, not drift.
|
|
96
|
+
return [];
|
|
97
|
+
}
|
|
98
|
+
try {
|
|
99
|
+
const row = db
|
|
100
|
+
.prepare("SELECT value FROM ItemTable WHERE key = 'composer.composerData'")
|
|
101
|
+
.get();
|
|
102
|
+
if (!row) {
|
|
103
|
+
// ACCEPTED_DEGRADATIONS.emptyWorkspace
|
|
104
|
+
return [];
|
|
105
|
+
}
|
|
106
|
+
let data;
|
|
107
|
+
try {
|
|
108
|
+
data = JSON.parse(row.value);
|
|
133
109
|
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
warnDrift(wsDbPath, `composer.composerData value is not valid JSON: ${err.message}`, 0);
|
|
112
|
+
return [];
|
|
113
|
+
}
|
|
114
|
+
if (data.allComposers !== undefined && !Array.isArray(data.allComposers)) {
|
|
115
|
+
warnDrift(wsDbPath, `expected 'allComposers' to be an array, got ${describeType(data.allComposers)}`, 0);
|
|
116
|
+
return [];
|
|
117
|
+
}
|
|
118
|
+
return data.allComposers ?? [];
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
// The ItemTable is a required workspace-storage contract — the db
|
|
122
|
+
// opened but a query against it failed, meaning Cursor's internal
|
|
123
|
+
// format changed. Throw so callers surface the drift instead of
|
|
124
|
+
// silently returning zero chunks.
|
|
125
|
+
throw new Error(`[${SCRAPER_NAME}] ItemTable unreadable at ${wsDbPath}: ${err.message}`);
|
|
126
|
+
}
|
|
127
|
+
finally {
|
|
128
|
+
db?.close();
|
|
134
129
|
}
|
|
135
130
|
}
|
|
136
|
-
|
|
137
|
-
|
|
131
|
+
*readComposerMessages(globalDb, composerRefs, since, wsPathForWarn) {
|
|
132
|
+
const getComposer = globalDb.prepare("SELECT value FROM cursorDiskKV WHERE key = ?");
|
|
133
|
+
const getBubble = globalDb.prepare("SELECT value FROM cursorDiskKV WHERE key = ?");
|
|
134
|
+
for (const ref of composerRefs) {
|
|
135
|
+
const composerRow = getComposer.get(`composerData:${ref.composerId}`);
|
|
136
|
+
if (!composerRow) {
|
|
137
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, "workspace references a composer that is missing from globalStorage", 0);
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
let composer;
|
|
138
141
|
try {
|
|
139
|
-
|
|
140
|
-
|
|
142
|
+
composer = JSON.parse(composerRow.value);
|
|
143
|
+
}
|
|
144
|
+
catch (err) {
|
|
145
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, `composer JSON not parseable: ${err.message}`, 0);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
// Strict-mode schema check: 'fullConversationHeadersOnly' is the
|
|
149
|
+
// required list of turns. If it's missing or renamed, emitting zero
|
|
150
|
+
// chunks would be silent data loss. Warn so drift is observable.
|
|
151
|
+
if (composer.fullConversationHeadersOnly === undefined) {
|
|
152
|
+
const suspiciousRename = Object.entries(composer).find(([, v]) => Array.isArray(v) &&
|
|
153
|
+
v.length > 0 &&
|
|
154
|
+
isRecord(v[0]) &&
|
|
155
|
+
"bubbleId" in v[0]);
|
|
156
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, suspiciousRename
|
|
157
|
+
? `'fullConversationHeadersOnly' missing; suspected rename to '${suspiciousRename[0]}'`
|
|
158
|
+
: "'fullConversationHeadersOnly' missing — composer has no turn list", 0);
|
|
159
|
+
continue;
|
|
160
|
+
}
|
|
161
|
+
if (!Array.isArray(composer.fullConversationHeadersOnly)) {
|
|
162
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, `expected 'fullConversationHeadersOnly' to be an array, got ` +
|
|
163
|
+
describeType(composer.fullConversationHeadersOnly), 0);
|
|
164
|
+
continue;
|
|
141
165
|
}
|
|
142
|
-
|
|
143
|
-
|
|
166
|
+
if (composer.modelConfig !== undefined && composer.modelConfig !== null &&
|
|
167
|
+
!isRecord(composer.modelConfig)) {
|
|
168
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, `expected 'modelConfig' to be object or absent, got ${describeType(composer.modelConfig)}`, 0);
|
|
169
|
+
}
|
|
170
|
+
else if (composer.modelConfig === null) {
|
|
171
|
+
warnDrift(`${wsPathForWarn}#composerData:${ref.composerId}`, "'modelConfig' is null — falling back to composerId as model label", 0);
|
|
172
|
+
}
|
|
173
|
+
const headers = composer.fullConversationHeadersOnly ?? [];
|
|
174
|
+
if (headers.length === 0)
|
|
175
|
+
continue;
|
|
176
|
+
const model = composer.modelConfig?.modelName ?? ref.composerId;
|
|
177
|
+
const composerMode = normalizeComposerMode(composer.unifiedMode ?? ref.unifiedMode);
|
|
178
|
+
const sessionId = ref.composerId;
|
|
179
|
+
let messageIndex = 0;
|
|
180
|
+
for (const header of headers) {
|
|
181
|
+
const bubbleRow = getBubble.get(`bubbleId:${ref.composerId}:${header.bubbleId}`);
|
|
182
|
+
if (!bubbleRow)
|
|
183
|
+
continue;
|
|
184
|
+
let bubble;
|
|
185
|
+
try {
|
|
186
|
+
bubble = JSON.parse(bubbleRow.value);
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const timestamp = toDate(bubble.createdAt);
|
|
192
|
+
if (timestamp <= since) {
|
|
193
|
+
messageIndex++;
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
const content = toNonEmptyString(bubble.text) ?? "";
|
|
197
|
+
if (!content) {
|
|
198
|
+
messageIndex++;
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
const role = normalizeRole(bubble.type);
|
|
202
|
+
yield {
|
|
203
|
+
tool: "cursor",
|
|
204
|
+
sessionId,
|
|
205
|
+
timestamp,
|
|
206
|
+
role,
|
|
207
|
+
content,
|
|
208
|
+
metadata: {
|
|
209
|
+
messageIndex,
|
|
210
|
+
tokenEstimate: estimateTokens(content),
|
|
211
|
+
referencedFiles: [],
|
|
212
|
+
model: bubble.modelInfo?.modelName ?? model,
|
|
213
|
+
composerMode,
|
|
214
|
+
},
|
|
215
|
+
};
|
|
216
|
+
messageIndex++;
|
|
144
217
|
}
|
|
145
218
|
}
|
|
146
|
-
return [];
|
|
147
219
|
}
|
|
148
|
-
async
|
|
220
|
+
async resolveWorkspaceDatabasePaths() {
|
|
149
221
|
try {
|
|
150
222
|
const target = await stat(this.cursorStorePath);
|
|
151
223
|
if (target.isFile()) {
|
|
@@ -158,27 +230,79 @@ export class CursorScraper {
|
|
|
158
230
|
catch {
|
|
159
231
|
return [];
|
|
160
232
|
}
|
|
161
|
-
const
|
|
162
|
-
const resolved = await Promise.all(patterns.map((pattern) => glob(pattern, {
|
|
233
|
+
const paths = await glob("**/state.vscdb", {
|
|
163
234
|
cwd: this.cursorStorePath,
|
|
164
235
|
absolute: true,
|
|
165
236
|
nodir: true,
|
|
166
|
-
})
|
|
167
|
-
|
|
237
|
+
});
|
|
238
|
+
if (!this.projectRoot) {
|
|
239
|
+
return paths;
|
|
240
|
+
}
|
|
241
|
+
const filtered = [];
|
|
242
|
+
for (const path of paths) {
|
|
243
|
+
if (await workspaceMatchesProject(path, this.projectRoot)) {
|
|
244
|
+
filtered.push(path);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
return filtered;
|
|
168
248
|
}
|
|
169
249
|
}
|
|
250
|
+
async function workspaceMatchesProject(workspaceDbPath, projectRoot) {
|
|
251
|
+
try {
|
|
252
|
+
const raw = await readFile(join(dirname(workspaceDbPath), "workspace.json"), "utf-8");
|
|
253
|
+
const parsed = JSON.parse(raw);
|
|
254
|
+
const folder = typeof parsed.folder === "string" ? parsed.folder : undefined;
|
|
255
|
+
if (!folder) {
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
const folderPath = folder.startsWith("file:") ? fileURLToPath(folder) : folder;
|
|
259
|
+
return pathMatchesProject(folderPath, projectRoot);
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Derive the Cursor global storage path from a workspace database path.
|
|
267
|
+
*
|
|
268
|
+
* Workspace path structure:
|
|
269
|
+
* .../Cursor/User/workspaceStorage/<hash>/state.vscdb
|
|
270
|
+
* Global storage:
|
|
271
|
+
* .../Cursor/User/globalStorage/state.vscdb
|
|
272
|
+
*/
|
|
273
|
+
function deriveGlobalStoragePath(workspaceDbPath) {
|
|
274
|
+
const normalized = workspaceDbPath.replace(/\\/g, "/");
|
|
275
|
+
const wsIdx = normalized.indexOf("/workspaceStorage/");
|
|
276
|
+
if (wsIdx === -1)
|
|
277
|
+
return null;
|
|
278
|
+
const userDir = normalized.slice(0, wsIdx);
|
|
279
|
+
return join(userDir, "globalStorage", "state.vscdb");
|
|
280
|
+
}
|
|
170
281
|
function normalizeRole(value) {
|
|
171
|
-
|
|
282
|
+
// Numeric type from bubble data: 1 = user, 2 = assistant.
|
|
283
|
+
if (typeof value === "number") {
|
|
284
|
+
if (value === BUBBLE_TYPE_USER)
|
|
285
|
+
return "user";
|
|
286
|
+
if (value === BUBBLE_TYPE_ASSISTANT)
|
|
287
|
+
return "assistant";
|
|
172
288
|
return "system";
|
|
173
289
|
}
|
|
174
|
-
|
|
290
|
+
// String role from legacy format.
|
|
291
|
+
const map = {
|
|
292
|
+
user: "user",
|
|
293
|
+
human: "user",
|
|
294
|
+
assistant: "assistant",
|
|
295
|
+
ai: "assistant",
|
|
296
|
+
system: "system",
|
|
297
|
+
tool: "tool",
|
|
298
|
+
};
|
|
299
|
+
return map[(value ?? "").toLowerCase()] ?? "system";
|
|
175
300
|
}
|
|
176
301
|
function normalizeComposerMode(value) {
|
|
177
302
|
return value === "agent" ? "agent" : "normal";
|
|
178
303
|
}
|
|
179
|
-
function
|
|
180
|
-
|
|
181
|
-
return file.replace(/\.[^.]+$/, "") || "unknown";
|
|
304
|
+
function isRecord(value) {
|
|
305
|
+
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
|
182
306
|
}
|
|
183
307
|
function toNonEmptyString(value) {
|
|
184
308
|
if (typeof value !== "string") {
|
|
@@ -187,32 +311,4 @@ function toNonEmptyString(value) {
|
|
|
187
311
|
const trimmed = value.trim();
|
|
188
312
|
return trimmed.length > 0 ? trimmed : undefined;
|
|
189
313
|
}
|
|
190
|
-
function toMessageIndex(value) {
|
|
191
|
-
const index = Number(value);
|
|
192
|
-
if (Number.isFinite(index) && index >= 0) {
|
|
193
|
-
return Math.floor(index);
|
|
194
|
-
}
|
|
195
|
-
return 0;
|
|
196
|
-
}
|
|
197
|
-
function coerceDate(value) {
|
|
198
|
-
if (value instanceof Date) {
|
|
199
|
-
return value;
|
|
200
|
-
}
|
|
201
|
-
if (typeof value === "number" && Number.isFinite(value)) {
|
|
202
|
-
const millis = value > 10_000_000_000 ? value : value * 1000;
|
|
203
|
-
return new Date(millis);
|
|
204
|
-
}
|
|
205
|
-
if (typeof value === "string" && value.length > 0) {
|
|
206
|
-
const numeric = Number(value);
|
|
207
|
-
if (Number.isFinite(numeric) && value.trim() !== "") {
|
|
208
|
-
const millis = numeric > 10_000_000_000 ? numeric : numeric * 1000;
|
|
209
|
-
return new Date(millis);
|
|
210
|
-
}
|
|
211
|
-
const parsed = new Date(value);
|
|
212
|
-
if (!Number.isNaN(parsed.getTime())) {
|
|
213
|
-
return parsed;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
return new Date(0);
|
|
217
|
-
}
|
|
218
314
|
//# sourceMappingURL=cursor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cursor.js","sourceRoot":"","sources":["../../../src/scrapers/cursor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"cursor.js","sourceRoot":"","sources":["../../../src/scrapers/cursor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,uDAAuD;AACvD,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEhC,MAAM,YAAY,GAAG,QAAQ,CAAC;AAE9B;;;GAGG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,iEAAiE;IACjE,gBAAgB,EAAE,qCAAqC;IACvD,2DAA2D;IAC3D,cAAc,EAAE,4CAA4C;IAC5D,wEAAwE;IACxE,YAAY,EAAE,8DAA8D;IAC5E,gDAAgD;IAChD,eAAe,EAAE,iCAAiC;IAClD,mEAAmE;IACnE,sBAAsB,EAAE,4CAA4C;CACrE,CAAC;AAEF,SAAS,SAAS,CAAC,UAAkB,EAAE,QAAgB,EAAE,eAAuB;IAC9E,OAAO,CAAC,IAAI,CACV,IAAI,YAAY,8BAA8B,UAAU,KAAK,QAAQ,GAAG;QACtE,sBAAsB,eAAe,GAAG,CAC3C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACzC,OAAO,OAAO,KAAK,CAAC;AACtB,CAAC;AAyBD,MAAM,OAAO,aAAc,SAAQ,eAA4B;IAI1C;IAEA;IALV,IAAI,GAAG,QAAQ,CAAC;IAEzB,YACmB,eAAuB,EACxC,QAAgB,EACC,WAAoB;QAErC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAJC,oBAAe,GAAf,eAAe,CAAQ;QAEvB,gBAAW,GAAX,WAAW,CAAS;IAGvC,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;QACzD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IAC1B,CAAC;IAED,aAAa;QACX,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,KAAY;QACxB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClD,MAAM,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,aAAa,CAAC;QAC5C,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,QAAQ;QACb,KAAK,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,CAAC,eAAe,CAAC,KAAW;QACxC,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,6BAA6B,EAAE,CAAC;QAElE,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;YACpC,MAAM,YAAY,GAAG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACzD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,MAAM,UAAU,GAAG,uBAAuB,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,GAA6B,IAAI,CAAC;YAC9C,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,QAAQ,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC7E,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,8DAA8D;gBAC9D,uEAAuE;gBACvE,SAAS,CACP,UAAU,EACV,6BAA8B,GAAa,CAAC,OAAO,EAAE,EACrD,YAAY,CAAC,MAAM,CACpB,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,QAAQ,EAAE,KAAK,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,sBAAsB,CAAC,QAAgB;QAC7C,IAAI,EAAE,GAA6B,IAAI,CAAC;QAExC,IAAI,CAAC;YACH,EAAE,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,oEAAoE;YACpE,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE;iBACX,OAAO,CAAC,iEAAiE,CAAC;iBAC1E,GAAG,EAAmC,CAAC;YAE1C,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,uCAAuC;gBACvC,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,IAAI,IAA+C,CAAC;YACpD,IAAI,CAAC;gBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAA8C,CAAC;YAC5E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,CACP,QAAQ,EACR,kDAAmD,GAAa,CAAC,OAAO,EAAE,EAC1E,CAAC,CACF,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;gBACzE,SAAS,CACP,QAAQ,EACR,+CAA+C,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,EAChF,CAAC,CACF,CAAC;gBACF,OAAO,EAAE,CAAC;YACZ,CAAC;YAED,OAAO,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QACjC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,kEAAkE;YAClE,kEAAkE;YAClE,gEAAgE;YAChE,kCAAkC;YAClC,MAAM,IAAI,KAAK,CACb,IAAI,YAAY,6BAA6B,QAAQ,KAAM,GAAa,CAAC,OAAO,EAAE,CACnF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,EAAE,EAAE,KAAK,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAEO,CAAC,oBAAoB,CAC3B,QAA2B,EAC3B,YAAoC,EACpC,KAAW,EACX,aAAqB;QAErB,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAClC,8CAA8C,CAC/C,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAChC,8CAA8C,CAC/C,CAAC;QAEF,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CACjC,gBAAgB,GAAG,CAAC,UAAU,EAAE,CACA,CAAC;YAEnC,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,oEAAoE,EACpE,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,QAA4B,CAAC;YACjC,IAAI,CAAC;gBACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,CAAuB,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,gCAAiC,GAAa,CAAC,OAAO,EAAE,EACxD,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,iEAAiE;YACjE,oEAAoE;YACpE,iEAAiE;YACjE,IAAI,QAAQ,CAAC,2BAA2B,KAAK,SAAS,EAAE,CAAC;gBACvD,MAAM,gBAAgB,GAAG,MAAM,CAAC,OAAO,CAAC,QAA8C,CAAC,CAAC,IAAI,CAC1F,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACR,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;oBAChB,CAAC,CAAC,MAAM,GAAG,CAAC;oBACZ,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACd,UAAU,IAAK,CAAC,CAAC,CAAC,CAA6B,CAClD,CAAC;gBACF,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,gBAAgB;oBACd,CAAC,CAAC,+DAA+D,gBAAgB,CAAC,CAAC,CAAC,GAAG;oBACvF,CAAC,CAAC,mEAAmE,EACvE,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EAAE,CAAC;gBACzD,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,6DAA6D;oBAC3D,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC,EACpD,CAAC,CACF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI;gBACnE,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACpC,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,sDAAsD,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,EAC1F,CAAC,CACF,CAAC;YACJ,CAAC;iBAAM,IAAI,QAAQ,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;gBACzC,SAAS,CACP,GAAG,aAAa,iBAAiB,GAAG,CAAC,UAAU,EAAE,EACjD,mEAAmE,EACnE,CAAC,CACF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,2BAA2B,IAAI,EAAE,CAAC;YAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEnC,MAAM,KAAK,GACT,QAAQ,CAAC,WAAW,EAAE,SAAS,IAAI,GAAG,CAAC,UAAU,CAAC;YACpD,MAAM,YAAY,GAAG,qBAAqB,CACxC,QAAQ,CAAC,WAAW,IAAI,GAAG,CAAC,WAAW,CACxC,CAAC;YACF,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC;YAEjC,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAC7B,YAAY,GAAG,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,EAAE,CACf,CAAC;gBAEnC,IAAI,CAAC,SAAS;oBAAE,SAAS;gBAEzB,IAAI,MAAwB,CAAC;gBAC7B,IAAI,CAAC;oBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAqB,CAAC;gBAC3D,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;gBAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,SAAS,IAAI,KAAK,EAAE,CAAC;oBACvB,YAAY,EAAE,CAAC;oBACf,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,YAAY,EAAE,CAAC;oBACf,SAAS;gBACX,CAAC;gBAED,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAExC,MAAM;oBACJ,IAAI,EAAE,QAAQ;oBACd,SAAS;oBACT,SAAS;oBACT,IAAI;oBACJ,OAAO;oBACP,QAAQ,EAAE;wBACR,YAAY;wBACZ,aAAa,EAAE,cAAc,CAAC,OAAO,CAAC;wBACtC,eAAe,EAAE,EAAE;wBACnB,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,SAAS,IAAI,KAAK;wBAC3C,YAAY;qBACb;iBACF,CAAC;gBACF,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,6BAA6B;QACzC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAChD,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpB,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAChC,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;gBAC1B,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE;YACzC,GAAG,EAAE,IAAI,CAAC,eAAe;YACzB,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,IAAI;SACZ,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,MAAM,uBAAuB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC1D,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAED,KAAK,UAAU,uBAAuB,CACpC,eAAuB,EACvB,WAAmB;IAEnB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,gBAAgB,CAAC,EAAE,OAAO,CAAC,CAAC;QACtF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;QAC1D,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7E,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/E,OAAO,kBAAkB,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,uBAAuB,CAAC,eAAuB;IACtD,MAAM,UAAU,GAAG,eAAe,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACvD,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAE9B,MAAM,OAAO,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC3C,OAAO,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,aAAa,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,aAAa,CAAC,KAAuB;IAC5C,0DAA0D;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,IAAI,KAAK,KAAK,gBAAgB;YAAE,OAAO,MAAM,CAAC;QAC9C,IAAI,KAAK,KAAK,qBAAqB;YAAE,OAAO,WAAW,CAAC;QACxD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,kCAAkC;IAClC,MAAM,GAAG,GAAwC;QAC/C,IAAI,EAAE,MAAM;QACZ,KAAK,EAAE,MAAM;QACb,SAAS,EAAE,WAAW;QACtB,EAAE,EAAE,WAAW;QACf,MAAM,EAAE,QAAQ;QAChB,IAAI,EAAE,MAAM;KACb,CAAC;IACF,OAAO,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,QAAQ,CAAC;AACtD,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;AAClD,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public API for building xtctx-compatible scrapers.
|
|
3
|
+
*
|
|
4
|
+
* External scraper packages should import from "xtctx/scrapers":
|
|
5
|
+
*
|
|
6
|
+
* ```ts
|
|
7
|
+
* import { AbstractScraper, estimateTokens } from "xtctx/scrapers";
|
|
8
|
+
* import type { ConversationChunk, ScraperState } from "xtctx/scrapers";
|
|
9
|
+
* ```
|
|
10
|
+
*
|
|
11
|
+
* See `AbstractScraper` for the full authoring guide.
|
|
12
|
+
*/
|
|
13
|
+
export type { ChunkMetadata, ConversationChunk, ConversationScraper, ScraperState, } from "../types/scraper.js";
|
|
14
|
+
export { AbstractScraper, ScraperStateManager, estimateTokens } from "./base.js";
|
|
15
|
+
export { AntigravityScraper } from "./antigravity.js";
|
|
16
|
+
export { OpenCodeScraper } from "./opencode.js";
|
|
17
|
+
export { CopilotCliScraper } from "./copilot-cli.js";
|
|
18
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/scrapers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,GACb,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public API for building xtctx-compatible scrapers.
|
|
3
|
+
*
|
|
4
|
+
* External scraper packages should import from "xtctx/scrapers":
|
|
5
|
+
*
|
|
6
|
+
* ```ts
|
|
7
|
+
* import { AbstractScraper, estimateTokens } from "xtctx/scrapers";
|
|
8
|
+
* import type { ConversationChunk, ScraperState } from "xtctx/scrapers";
|
|
9
|
+
* ```
|
|
10
|
+
*
|
|
11
|
+
* See `AbstractScraper` for the full authoring guide.
|
|
12
|
+
*/
|
|
13
|
+
export { AbstractScraper, ScraperStateManager, estimateTokens } from "./base.js";
|
|
14
|
+
export { AntigravityScraper } from "./antigravity.js";
|
|
15
|
+
export { OpenCodeScraper } from "./opencode.js";
|
|
16
|
+
export { CopilotCliScraper } from "./copilot-cli.js";
|
|
17
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scrapers/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AASH,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { OpenCodeChunk } from "../types/scraper.js";
|
|
2
|
+
import { AbstractScraper } from "./base.js";
|
|
3
|
+
/**
|
|
4
|
+
* Mutation shapes the opencode scraper tolerates silently. Anything outside
|
|
5
|
+
* this whitelist that drops records must warn (or throw for required tables).
|
|
6
|
+
*/
|
|
7
|
+
export declare const ACCEPTED_DEGRADATIONS: {
|
|
8
|
+
/** opencode.db missing — opencode CLI not installed on this machine. */
|
|
9
|
+
missingDatabase: string;
|
|
10
|
+
/** better-sqlite3 native module unavailable — opt-in peer dep. */
|
|
11
|
+
missingSqliteBinding: string;
|
|
12
|
+
/** Sessions table empty — pristine opencode install. */
|
|
13
|
+
emptySessions: string;
|
|
14
|
+
/** A part with no extractable text (file, snapshot, step events). */
|
|
15
|
+
nonTextPart: string;
|
|
16
|
+
/** Reasoning parts on assistant messages — internal model thoughts; skipped by default. */
|
|
17
|
+
reasoningPart: string;
|
|
18
|
+
/** Forward-compat unknown keys alongside known fields. */
|
|
19
|
+
unknownFieldsAlongside: string;
|
|
20
|
+
/** A message data row that fails to parse as JSON — skip with warn. */
|
|
21
|
+
malformedMessageData: string;
|
|
22
|
+
/** A part data row that fails to parse as JSON — skip with warn. */
|
|
23
|
+
malformedPartData: string;
|
|
24
|
+
};
|
|
25
|
+
export declare class OpenCodeScraper extends AbstractScraper<OpenCodeChunk> {
|
|
26
|
+
private readonly opencodeDbPath;
|
|
27
|
+
readonly tool = "opencode";
|
|
28
|
+
constructor(opencodeDbPath: string, stateDir: string);
|
|
29
|
+
detect(): Promise<boolean>;
|
|
30
|
+
getStorePaths(): string[];
|
|
31
|
+
scrape(since?: Date): AsyncIterable<OpenCodeChunk>;
|
|
32
|
+
fullSync(): AsyncIterable<OpenCodeChunk>;
|
|
33
|
+
private readAllSessions;
|
|
34
|
+
private readFromDb;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=opencode.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"opencode.d.ts","sourceRoot":"","sources":["../../../src/scrapers/opencode.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,eAAe,EAA0B,MAAM,WAAW,CAAC;AAIpE;;;GAGG;AACH,eAAO,MAAM,qBAAqB;IAChC,wEAAwE;;IAExE,kEAAkE;;IAElE,wDAAwD;;IAExD,qEAAqE;;IAErE,2FAA2F;;IAE3F,0DAA0D;;IAE1D,uEAAuE;;IAEvE,oEAAoE;;CAErE,CAAC;AA2CF,qBAAa,eAAgB,SAAQ,eAAe,CAAC,aAAa,CAAC;IAI/D,OAAO,CAAC,QAAQ,CAAC,cAAc;IAHjC,QAAQ,CAAC,IAAI,cAAc;gBAGR,cAAc,EAAE,MAAM,EACvC,QAAQ,EAAE,MAAM;IAKZ,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;IAShC,aAAa,IAAI,MAAM,EAAE;IAIlB,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,GAAG,aAAa,CAAC,aAAa,CAAC;IAMlD,QAAQ,IAAI,aAAa,CAAC,aAAa,CAAC;YAIhC,eAAe;IAyC9B,OAAO,CAAE,UAAU;CAsLpB"}
|