skill-any-code 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/command.schemas.js +18 -0
- package/dist/application/analysis.app.service.js +264 -0
- package/dist/application/bootstrap.js +21 -0
- package/dist/application/services/llm.analysis.service.js +170 -0
- package/dist/common/config.js +213 -0
- package/dist/common/constants.js +11 -0
- package/dist/common/errors.js +37 -0
- package/dist/common/logger.js +77 -0
- package/dist/common/types.js +2 -0
- package/dist/common/ui.js +201 -0
- package/dist/common/utils.js +117 -0
- package/dist/domain/index.js +17 -0
- package/dist/domain/interfaces.js +2 -0
- package/dist/domain/services/analysis.service.js +696 -0
- package/dist/domain/services/incremental.service.js +81 -0
- package/dist/infrastructure/blacklist.service.js +71 -0
- package/dist/infrastructure/cache/file.hash.cache.js +140 -0
- package/dist/infrastructure/git/git.service.js +159 -0
- package/dist/infrastructure/git.service.js +157 -0
- package/dist/infrastructure/index.service.js +108 -0
- package/dist/infrastructure/llm/llm.usage.tracker.js +58 -0
- package/dist/infrastructure/llm/openai.client.js +141 -0
- package/{src/infrastructure/llm/prompt.template.ts → dist/infrastructure/llm/prompt.template.js} +31 -36
- package/dist/infrastructure/llm.service.js +61 -0
- package/dist/infrastructure/skill/skill.generator.js +83 -0
- package/{src/infrastructure/skill/templates/resolve.script.ts → dist/infrastructure/skill/templates/resolve.script.js} +18 -15
- package/dist/infrastructure/skill/templates/skill.md.template.js +47 -0
- package/dist/infrastructure/splitter/code.splitter.js +137 -0
- package/dist/infrastructure/storage.service.js +409 -0
- package/dist/infrastructure/worker-pool/parse.worker.impl.js +137 -0
- package/dist/infrastructure/worker-pool/parse.worker.js +43 -0
- package/dist/infrastructure/worker-pool/worker-pool.service.js +171 -0
- package/package.json +5 -1
- package/jest.config.js +0 -27
- package/src/adapters/command.schemas.ts +0 -21
- package/src/application/analysis.app.service.ts +0 -272
- package/src/application/bootstrap.ts +0 -35
- package/src/application/services/llm.analysis.service.ts +0 -237
- package/src/cli.ts +0 -297
- package/src/common/config.ts +0 -209
- package/src/common/constants.ts +0 -8
- package/src/common/errors.ts +0 -34
- package/src/common/logger.ts +0 -82
- package/src/common/types.ts +0 -385
- package/src/common/ui.ts +0 -228
- package/src/common/utils.ts +0 -81
- package/src/domain/index.ts +0 -1
- package/src/domain/interfaces.ts +0 -188
- package/src/domain/services/analysis.service.ts +0 -735
- package/src/domain/services/incremental.service.ts +0 -50
- package/src/index.ts +0 -6
- package/src/infrastructure/blacklist.service.ts +0 -37
- package/src/infrastructure/cache/file.hash.cache.ts +0 -119
- package/src/infrastructure/git/git.service.ts +0 -120
- package/src/infrastructure/git.service.ts +0 -121
- package/src/infrastructure/index.service.ts +0 -94
- package/src/infrastructure/llm/llm.usage.tracker.ts +0 -65
- package/src/infrastructure/llm/openai.client.ts +0 -162
- package/src/infrastructure/llm.service.ts +0 -70
- package/src/infrastructure/skill/skill.generator.ts +0 -53
- package/src/infrastructure/skill/templates/skill.md.template.ts +0 -45
- package/src/infrastructure/splitter/code.splitter.ts +0 -176
- package/src/infrastructure/storage.service.ts +0 -413
- package/src/infrastructure/worker-pool/parse.worker.impl.ts +0 -135
- package/src/infrastructure/worker-pool/parse.worker.ts +0 -9
- package/src/infrastructure/worker-pool/worker-pool.service.ts +0 -173
- package/tsconfig.json +0 -24
- package/tsconfig.test.json +0 -5
|
@@ -0,0 +1,696 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.AnalysisService = void 0;
|
|
40
|
+
const fs = __importStar(require("fs-extra"));
|
|
41
|
+
const path = __importStar(require("path"));
|
|
42
|
+
const crypto_1 = require("crypto");
|
|
43
|
+
const errors_1 = require("../../common/errors");
|
|
44
|
+
const logger_1 = require("../../common/logger");
|
|
45
|
+
const utils_1 = require("../../common/utils");
|
|
46
|
+
const openai_client_1 = require("../../infrastructure/llm/openai.client");
|
|
47
|
+
const llm_usage_tracker_1 = require("../../infrastructure/llm/llm.usage.tracker");
|
|
48
|
+
const code_splitter_1 = require("../../infrastructure/splitter/code.splitter");
|
|
49
|
+
const file_hash_cache_1 = require("../../infrastructure/cache/file.hash.cache");
|
|
50
|
+
const llm_analysis_service_1 = require("../../application/services/llm.analysis.service");
|
|
51
|
+
const worker_pool_service_1 = require("../../infrastructure/worker-pool/worker-pool.service");
|
|
52
|
+
const os_1 = __importDefault(require("os"));
|
|
53
|
+
class AnalysisService {
|
|
54
|
+
gitService;
|
|
55
|
+
storageService;
|
|
56
|
+
blacklistService;
|
|
57
|
+
projectSlug;
|
|
58
|
+
currentCommit;
|
|
59
|
+
llmConfig;
|
|
60
|
+
onTokenUsageSnapshot;
|
|
61
|
+
llmAnalysisService;
|
|
62
|
+
tracker;
|
|
63
|
+
constructor(gitService, storageService, blacklistService, projectSlug, currentCommit, llmConfig, onTokenUsageSnapshot) {
|
|
64
|
+
this.gitService = gitService;
|
|
65
|
+
this.storageService = storageService;
|
|
66
|
+
this.blacklistService = blacklistService;
|
|
67
|
+
this.projectSlug = projectSlug;
|
|
68
|
+
this.currentCommit = currentCommit;
|
|
69
|
+
this.llmConfig = llmConfig;
|
|
70
|
+
this.onTokenUsageSnapshot = onTokenUsageSnapshot;
|
|
71
|
+
this.tracker = new llm_usage_tracker_1.LLMUsageTracker(this.onTokenUsageSnapshot);
|
|
72
|
+
const llmClient = new openai_client_1.OpenAIClient(llmConfig, this.tracker);
|
|
73
|
+
const fileSplitter = new code_splitter_1.CodeSplitter(llmClient);
|
|
74
|
+
const homeDir = os_1.default.homedir();
|
|
75
|
+
const resolvedCacheDir = llmConfig.cache_dir.replace(/^~(?=\/|\\|$)/, homeDir);
|
|
76
|
+
const cache = new file_hash_cache_1.FileHashCache({
|
|
77
|
+
cacheDir: resolvedCacheDir,
|
|
78
|
+
maxSizeMb: llmConfig.cache_max_size_mb,
|
|
79
|
+
});
|
|
80
|
+
this.llmAnalysisService = new llm_analysis_service_1.LLMAnalysisService(llmClient, fileSplitter, cache, llmConfig);
|
|
81
|
+
}
|
|
82
|
+
getTokenUsage() {
|
|
83
|
+
return this.tracker.getStats();
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* 统计将参与解析的对象总数(文件+目录),用于进度条 total。
|
|
87
|
+
* 与 fullAnalysis 使用相同的深度与黑名单规则。
|
|
88
|
+
*/
|
|
89
|
+
async countObjects(projectRoot, depth = -1) {
|
|
90
|
+
const rootStat = await fs.stat(projectRoot);
|
|
91
|
+
if (rootStat.isFile())
|
|
92
|
+
return 1;
|
|
93
|
+
let count = 0;
|
|
94
|
+
const walk = async (dirPath, currentDepth) => {
|
|
95
|
+
if (depth >= 1 && currentDepth > depth) {
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
const entries = await fs.readdir(dirPath, { withFileTypes: true });
|
|
99
|
+
const valid = entries.filter(entry => {
|
|
100
|
+
const fullPath = path.join(dirPath, entry.name);
|
|
101
|
+
const relativePath = path.relative(projectRoot, fullPath);
|
|
102
|
+
const key = entry.isDirectory() ? `${relativePath}/` : relativePath;
|
|
103
|
+
return !this.blacklistService.isIgnored(key);
|
|
104
|
+
});
|
|
105
|
+
let hasContent = false;
|
|
106
|
+
for (const entry of valid) {
|
|
107
|
+
if (entry.isFile()) {
|
|
108
|
+
count++;
|
|
109
|
+
hasContent = true;
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (entry.isDirectory()) {
|
|
113
|
+
const childHas = await walk(path.join(dirPath, entry.name), currentDepth + 1);
|
|
114
|
+
if (childHas) {
|
|
115
|
+
hasContent = true;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
if (hasContent) {
|
|
120
|
+
count++;
|
|
121
|
+
}
|
|
122
|
+
return hasContent;
|
|
123
|
+
};
|
|
124
|
+
await walk(projectRoot, 1);
|
|
125
|
+
return count;
|
|
126
|
+
}
|
|
127
|
+
// ---------------------------------------------------------------------------
|
|
128
|
+
// 私有工具方法
|
|
129
|
+
// ---------------------------------------------------------------------------
|
|
130
|
+
async getFileGitMeta(projectRoot, relPath) {
|
|
131
|
+
const hasGitMeta = typeof this.gitService.getFileLastCommit === 'function' &&
|
|
132
|
+
typeof this.gitService.isFileDirty === 'function';
|
|
133
|
+
const fileGitCommitId = hasGitMeta
|
|
134
|
+
? await this.gitService.getFileLastCommit(projectRoot, relPath)
|
|
135
|
+
: null;
|
|
136
|
+
const isDirty = hasGitMeta
|
|
137
|
+
? await this.gitService.isFileDirty(projectRoot, relPath)
|
|
138
|
+
: false;
|
|
139
|
+
return { fileGitCommitId, isDirty };
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Phase 1:遍历目录树,构建完整的任务图。
|
|
143
|
+
* 全量和增量共享此扫描逻辑,黑名单和深度限制在此阶段统一应用。
|
|
144
|
+
*/
|
|
145
|
+
async scanProjectTree(projectRoot, depth, onScanProgress) {
|
|
146
|
+
const depthEnabled = depth !== undefined && depth >= 1;
|
|
147
|
+
const maxDepth = depthEnabled ? depth : Number.POSITIVE_INFINITY;
|
|
148
|
+
const dirNodes = new Map();
|
|
149
|
+
const fileAbsByRel = new Map();
|
|
150
|
+
let scannedObjectCount = 0;
|
|
151
|
+
const rootRel = '.';
|
|
152
|
+
dirNodes.set(rootRel, {
|
|
153
|
+
absPath: projectRoot,
|
|
154
|
+
relPath: rootRel,
|
|
155
|
+
depth: 1,
|
|
156
|
+
childDirs: [],
|
|
157
|
+
childFiles: [],
|
|
158
|
+
});
|
|
159
|
+
const queue = [
|
|
160
|
+
{ rel: rootRel, abs: projectRoot, depth: 1 },
|
|
161
|
+
];
|
|
162
|
+
const scanConcurrency = Math.max(1, Math.min(8, os_1.default.cpus()?.length || 4));
|
|
163
|
+
const processDir = async (current) => {
|
|
164
|
+
const node = dirNodes.get(current.rel);
|
|
165
|
+
if (!node)
|
|
166
|
+
return;
|
|
167
|
+
if (current.depth > maxDepth)
|
|
168
|
+
return;
|
|
169
|
+
const entries = await fs.readdir(current.abs, { withFileTypes: true });
|
|
170
|
+
const validEntries = entries
|
|
171
|
+
.filter(entry => {
|
|
172
|
+
const fullPath = path.join(current.abs, entry.name);
|
|
173
|
+
const relativePath = path.relative(projectRoot, fullPath);
|
|
174
|
+
const key = entry.isDirectory() ? `${relativePath}/` : relativePath;
|
|
175
|
+
return !this.blacklistService.isIgnored(key);
|
|
176
|
+
})
|
|
177
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
178
|
+
for (const entry of validEntries) {
|
|
179
|
+
const fullPath = path.join(current.abs, entry.name);
|
|
180
|
+
const relPath = path.relative(projectRoot, fullPath) || entry.name;
|
|
181
|
+
if (entry.isFile()) {
|
|
182
|
+
node.childFiles.push(relPath);
|
|
183
|
+
fileAbsByRel.set(relPath, fullPath);
|
|
184
|
+
scannedObjectCount++;
|
|
185
|
+
if (scannedObjectCount % 10 === 0)
|
|
186
|
+
onScanProgress?.(scannedObjectCount);
|
|
187
|
+
}
|
|
188
|
+
else if (entry.isDirectory()) {
|
|
189
|
+
node.childDirs.push(relPath);
|
|
190
|
+
const childDepth = current.depth + 1;
|
|
191
|
+
dirNodes.set(relPath, {
|
|
192
|
+
absPath: fullPath,
|
|
193
|
+
relPath: relPath,
|
|
194
|
+
depth: childDepth,
|
|
195
|
+
childDirs: [],
|
|
196
|
+
childFiles: [],
|
|
197
|
+
});
|
|
198
|
+
queue.push({ rel: relPath, abs: fullPath, depth: childDepth });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
const runScanQueue = async () => {
|
|
203
|
+
const workers = Array.from({ length: scanConcurrency }, async () => {
|
|
204
|
+
// eslint-disable-next-line no-constant-condition
|
|
205
|
+
while (true) {
|
|
206
|
+
const current = queue.shift();
|
|
207
|
+
if (!current)
|
|
208
|
+
return;
|
|
209
|
+
await processDir(current);
|
|
210
|
+
}
|
|
211
|
+
});
|
|
212
|
+
await Promise.all(workers);
|
|
213
|
+
};
|
|
214
|
+
await runScanQueue();
|
|
215
|
+
// 目录剪枝:自底向上移除空目录
|
|
216
|
+
const allScannedDirs = Array.from(dirNodes.values());
|
|
217
|
+
const scannedDirsByDepthDesc = allScannedDirs
|
|
218
|
+
.slice()
|
|
219
|
+
.sort((a, b) => {
|
|
220
|
+
if (b.depth !== a.depth)
|
|
221
|
+
return b.depth - a.depth;
|
|
222
|
+
return a.relPath.localeCompare(b.relPath);
|
|
223
|
+
});
|
|
224
|
+
const keptDirs = new Set();
|
|
225
|
+
for (const d of scannedDirsByDepthDesc) {
|
|
226
|
+
const node = dirNodes.get(d.relPath);
|
|
227
|
+
if (!node)
|
|
228
|
+
continue;
|
|
229
|
+
node.childDirs = node.childDirs.filter(child => keptDirs.has(child));
|
|
230
|
+
const hasContent = node.childFiles.length > 0 || node.childDirs.length > 0;
|
|
231
|
+
if (hasContent) {
|
|
232
|
+
keptDirs.add(d.relPath);
|
|
233
|
+
scannedObjectCount++;
|
|
234
|
+
if (scannedObjectCount % 10 === 0)
|
|
235
|
+
onScanProgress?.(scannedObjectCount);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (scannedObjectCount > 0 && scannedObjectCount % 10 !== 0) {
|
|
239
|
+
onScanProgress?.(scannedObjectCount);
|
|
240
|
+
}
|
|
241
|
+
return { dirNodes, fileAbsByRel, keptDirs };
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* 增量专用:扫描存储目录,找出"有解析结果但对应源文件/目录已不存在"的孤立条目并清理。
|
|
245
|
+
*/
|
|
246
|
+
async cleanOrphanedResults(storageRoot, projectRoot, currentFileRels, keptDirs, removedSourcePaths) {
|
|
247
|
+
logger_1.logger.info('Scanning storage directory for orphaned result files...');
|
|
248
|
+
// 构建当前源码树的预期结果路径集合
|
|
249
|
+
const expectedResultPaths = new Set();
|
|
250
|
+
for (const relPath of currentFileRels.keys()) {
|
|
251
|
+
expectedResultPaths.add(path.resolve((0, utils_1.getFileOutputPath)(storageRoot, relPath)));
|
|
252
|
+
}
|
|
253
|
+
for (const dirRel of keptDirs) {
|
|
254
|
+
expectedResultPaths.add(path.resolve((0, utils_1.getDirOutputPath)(storageRoot, dirRel)));
|
|
255
|
+
}
|
|
256
|
+
const orphaned = [];
|
|
257
|
+
const walk = async (dirAbs) => {
|
|
258
|
+
if (!(await fs.pathExists(dirAbs)))
|
|
259
|
+
return;
|
|
260
|
+
const entries = await fs.readdir(dirAbs, { withFileTypes: true });
|
|
261
|
+
for (const entry of entries) {
|
|
262
|
+
const fullPath = path.join(dirAbs, entry.name);
|
|
263
|
+
if (entry.isDirectory()) {
|
|
264
|
+
if (entry.name.startsWith('.'))
|
|
265
|
+
continue;
|
|
266
|
+
await walk(fullPath);
|
|
267
|
+
}
|
|
268
|
+
else if (entry.isFile() && entry.name.endsWith('.md')) {
|
|
269
|
+
if (!expectedResultPaths.has(path.resolve(fullPath))) {
|
|
270
|
+
orphaned.push(fullPath);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
await walk(storageRoot);
|
|
276
|
+
if (orphaned.length > 0) {
|
|
277
|
+
logger_1.logger.info(`Found ${orphaned.length} orphaned result file(s). Cleaning up...`);
|
|
278
|
+
for (const p of orphaned) {
|
|
279
|
+
try {
|
|
280
|
+
const content = await fs.readFile(p, 'utf-8');
|
|
281
|
+
const match = content.match(/(?:^|\n)-\s*(?:Path|路径)\s*[::]\s*(.+)\s*$/m);
|
|
282
|
+
const sourcePath = match?.[1]?.trim();
|
|
283
|
+
if (sourcePath) {
|
|
284
|
+
removedSourcePaths.push(path.resolve(projectRoot, sourcePath));
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
catch { /* 无法读取则仅删除 */ }
|
|
288
|
+
await fs.remove(p);
|
|
289
|
+
}
|
|
290
|
+
// 自底向上清理因删除 .md 而变为空的存储子目录
|
|
291
|
+
const resolvedStorageRoot = path.resolve(storageRoot);
|
|
292
|
+
const removeEmptyDirs = async (dirAbs) => {
|
|
293
|
+
if (!(await fs.pathExists(dirAbs)))
|
|
294
|
+
return true;
|
|
295
|
+
const entries = await fs.readdir(dirAbs, { withFileTypes: true });
|
|
296
|
+
for (const entry of entries) {
|
|
297
|
+
const fullPath = path.join(dirAbs, entry.name);
|
|
298
|
+
if (entry.isDirectory() && !entry.name.startsWith('.')) {
|
|
299
|
+
await removeEmptyDirs(fullPath);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
// 重新读取:子目录可能刚被删除
|
|
303
|
+
const remaining = await fs.readdir(dirAbs);
|
|
304
|
+
if (remaining.length === 0 && path.resolve(dirAbs) !== resolvedStorageRoot) {
|
|
305
|
+
await fs.remove(dirAbs);
|
|
306
|
+
return true;
|
|
307
|
+
}
|
|
308
|
+
return false;
|
|
309
|
+
};
|
|
310
|
+
await removeEmptyDirs(storageRoot);
|
|
311
|
+
}
|
|
312
|
+
else {
|
|
313
|
+
logger_1.logger.info('No orphaned result files found');
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
// ---------------------------------------------------------------------------
|
|
317
|
+
// 统一解析入口
|
|
318
|
+
// ---------------------------------------------------------------------------
|
|
319
|
+
async analyze(params) {
|
|
320
|
+
const startTime = Date.now();
|
|
321
|
+
const errors = [];
|
|
322
|
+
const completedFiles = [];
|
|
323
|
+
const completedDirs = [];
|
|
324
|
+
const indexEntries = [];
|
|
325
|
+
const removedSourcePaths = [];
|
|
326
|
+
const storageRoot = this.storageService.getStoragePath(this.projectSlug);
|
|
327
|
+
// --- 单文件特殊处理 ---
|
|
328
|
+
const rootStat = await fs.stat(params.projectRoot);
|
|
329
|
+
if (rootStat.isFile()) {
|
|
330
|
+
try {
|
|
331
|
+
const content = await fs.readFile(params.projectRoot, 'utf-8');
|
|
332
|
+
const fileHash = (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
333
|
+
const parseResult = await this.llmAnalysisService.analyzeFile(params.projectRoot, content, fileHash);
|
|
334
|
+
const relativePath = path.basename(params.projectRoot);
|
|
335
|
+
const fileResult = {
|
|
336
|
+
...parseResult,
|
|
337
|
+
path: relativePath,
|
|
338
|
+
commitHash: params.commitHash,
|
|
339
|
+
};
|
|
340
|
+
await this.storageService.saveFileAnalysis(this.projectSlug, relativePath, fileResult);
|
|
341
|
+
completedFiles.push(relativePath);
|
|
342
|
+
params.onTotalKnown?.(1);
|
|
343
|
+
params.onObjectPlanned?.({ type: 'file', path: relativePath });
|
|
344
|
+
params.onObjectCompleted?.({ type: 'file', path: relativePath }, { status: 'parsed' });
|
|
345
|
+
const sourceAbsPath = path.resolve(params.projectRoot);
|
|
346
|
+
const resultAbsPath = path.resolve(storageRoot, (0, utils_1.getFileOutputPath)(storageRoot, relativePath));
|
|
347
|
+
indexEntries.push({ sourcePath: sourceAbsPath, resultPath: resultAbsPath, type: 'file' });
|
|
348
|
+
}
|
|
349
|
+
catch (e) {
|
|
350
|
+
errors.push({ path: params.projectRoot, message: e.message });
|
|
351
|
+
}
|
|
352
|
+
return {
|
|
353
|
+
success: errors.length === 0,
|
|
354
|
+
analyzedFilesCount: completedFiles.length,
|
|
355
|
+
analyzedDirsCount: 0,
|
|
356
|
+
duration: Date.now() - startTime,
|
|
357
|
+
errors,
|
|
358
|
+
projectSlug: this.projectSlug,
|
|
359
|
+
summaryPath: path.join(storageRoot, 'index.md'),
|
|
360
|
+
indexEntries,
|
|
361
|
+
removedSourcePaths: [],
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
// ===================================================================
|
|
365
|
+
// Phase 1:统一遍历目录树
|
|
366
|
+
// ===================================================================
|
|
367
|
+
logger_1.logger.debug('Phase 1: scanning directory tree...');
|
|
368
|
+
const { dirNodes, fileAbsByRel, keptDirs } = await this.scanProjectTree(params.projectRoot, params.depth, params.onScanProgress);
|
|
369
|
+
logger_1.logger.debug(`Scan completed: ${fileAbsByRel.size} file(s), ${keptDirs.size} non-empty directory(ies)`);
|
|
370
|
+
// 清理被剪枝(空)目录的残留结果文件
|
|
371
|
+
for (const d of dirNodes.values()) {
|
|
372
|
+
if (!keptDirs.has(d.relPath) && d.relPath !== '.') {
|
|
373
|
+
const out = (0, utils_1.getDirOutputPath)(storageRoot, d.relPath);
|
|
374
|
+
if (await fs.pathExists(out)) {
|
|
375
|
+
await fs.remove(out);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
// ===================================================================
|
|
380
|
+
// Phase 2:应用文件过滤策略,构建文件任务队列
|
|
381
|
+
// ===================================================================
|
|
382
|
+
logger_1.logger.debug(`Phase 2: applying file filter (mode=${params.mode})...`);
|
|
383
|
+
const includedFiles = new Set();
|
|
384
|
+
const filterConcurrency = Math.max(1, Math.min(8, os_1.default.cpus()?.length || 4));
|
|
385
|
+
await (0, utils_1.mapLimit)(Array.from(fileAbsByRel.entries()), filterConcurrency, async ([relPath, absPath]) => {
|
|
386
|
+
if (await params.fileFilter(relPath, absPath)) {
|
|
387
|
+
includedFiles.add(relPath);
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
logger_1.logger.debug(`File filtering done: ${includedFiles.size}/${fileAbsByRel.size} file(s) queued`);
|
|
391
|
+
// ===================================================================
|
|
392
|
+
// Phase 2.5:增量模式 — 清理孤立的解析结果
|
|
393
|
+
// ===================================================================
|
|
394
|
+
if (params.mode === 'incremental') {
|
|
395
|
+
await this.cleanOrphanedResults(storageRoot, params.projectRoot, fileAbsByRel, keptDirs, removedSourcePaths);
|
|
396
|
+
}
|
|
397
|
+
// ===================================================================
|
|
398
|
+
// Phase 3:构建目录任务队列
|
|
399
|
+
// 规则:
|
|
400
|
+
// - 至少有 1 个子项(文件或子目录)在任务队列中 → 目录进入队列
|
|
401
|
+
// - 目录自身的结果 md 缺失 → 目录进入队列
|
|
402
|
+
// - 自底向上传播:底层文件变更会驱动整条祖先链重新聚合
|
|
403
|
+
// ===================================================================
|
|
404
|
+
logger_1.logger.debug('Phase 3: building directory task queue...');
|
|
405
|
+
const includedDirs = new Set();
|
|
406
|
+
const allKeptDirsSorted = Array.from(dirNodes.values())
|
|
407
|
+
.filter(d => keptDirs.has(d.relPath))
|
|
408
|
+
.sort((a, b) => {
|
|
409
|
+
if (b.depth !== a.depth)
|
|
410
|
+
return b.depth - a.depth;
|
|
411
|
+
return a.relPath.localeCompare(b.relPath);
|
|
412
|
+
});
|
|
413
|
+
for (const dir of allKeptDirsSorted) {
|
|
414
|
+
let shouldInclude = dir.childFiles.some(f => includedFiles.has(f)) ||
|
|
415
|
+
dir.childDirs.some(d => includedDirs.has(d));
|
|
416
|
+
if (!shouldInclude) {
|
|
417
|
+
const dirMdPath = (0, utils_1.getDirOutputPath)(storageRoot, dir.relPath);
|
|
418
|
+
shouldInclude = !(await fs.pathExists(dirMdPath));
|
|
419
|
+
}
|
|
420
|
+
if (shouldInclude) {
|
|
421
|
+
includedDirs.add(dir.relPath);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
logger_1.logger.debug(`Directory filtering done: ${includedDirs.size}/${keptDirs.size} directory(ies) queued`);
|
|
425
|
+
// ===================================================================
|
|
426
|
+
// Phase 4:排序 + 通知总数
|
|
427
|
+
// ===================================================================
|
|
428
|
+
const plannedFiles = Array.from(includedFiles).sort((a, b) => a.localeCompare(b));
|
|
429
|
+
const plannedDirs = allKeptDirsSorted
|
|
430
|
+
.filter(d => includedDirs.has(d.relPath))
|
|
431
|
+
.map(d => d.relPath);
|
|
432
|
+
const totalObjects = plannedFiles.length + plannedDirs.length;
|
|
433
|
+
params.onTotalKnown?.(totalObjects);
|
|
434
|
+
for (const f of plannedFiles) {
|
|
435
|
+
params.onObjectPlanned?.({ type: 'file', path: f });
|
|
436
|
+
}
|
|
437
|
+
for (const d of plannedDirs) {
|
|
438
|
+
params.onObjectPlanned?.({ type: 'directory', path: d });
|
|
439
|
+
}
|
|
440
|
+
if (totalObjects === 0) {
|
|
441
|
+
logger_1.logger.info('No objects to (re)analyze');
|
|
442
|
+
return {
|
|
443
|
+
success: true,
|
|
444
|
+
analyzedFilesCount: 0,
|
|
445
|
+
analyzedDirsCount: 0,
|
|
446
|
+
duration: Date.now() - startTime,
|
|
447
|
+
errors: [],
|
|
448
|
+
projectSlug: this.projectSlug,
|
|
449
|
+
summaryPath: path.join(storageRoot, 'index.md'),
|
|
450
|
+
indexEntries: [],
|
|
451
|
+
removedSourcePaths,
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
// ===================================================================
|
|
455
|
+
// Phase 5:统一执行管线 — 文件解析 + 目录聚合
|
|
456
|
+
// ===================================================================
|
|
457
|
+
const workerPool = new worker_pool_service_1.WorkerPoolService(this.llmConfig, params.concurrency);
|
|
458
|
+
const fileResults = new Map();
|
|
459
|
+
const dirResults = new Map();
|
|
460
|
+
try {
|
|
461
|
+
// --- 5a:文件解析 ---
|
|
462
|
+
await (0, utils_1.mapLimit)(plannedFiles, params.concurrency, async (relPath) => {
|
|
463
|
+
const fileObj = { type: 'file', path: relPath };
|
|
464
|
+
try {
|
|
465
|
+
const absPath = fileAbsByRel.get(relPath);
|
|
466
|
+
const content = await fs.readFile(absPath, 'utf-8');
|
|
467
|
+
const fileHash = (0, crypto_1.createHash)('sha256').update(content).digest('hex');
|
|
468
|
+
const { fileGitCommitId, isDirty } = await this.getFileGitMeta(params.projectRoot, relPath);
|
|
469
|
+
params.onObjectStarted?.(fileObj);
|
|
470
|
+
const workerRes = await workerPool.submitFileAnalysisTask(absPath, content, fileHash);
|
|
471
|
+
const parseResult = workerRes?.analysis ?? workerRes;
|
|
472
|
+
if (workerRes?.usage) {
|
|
473
|
+
this.tracker.addTotals(workerRes.usage);
|
|
474
|
+
}
|
|
475
|
+
const fileResult = {
|
|
476
|
+
...parseResult,
|
|
477
|
+
path: relPath,
|
|
478
|
+
commitHash: params.commitHash,
|
|
479
|
+
fileGitCommitId: fileGitCommitId ?? undefined,
|
|
480
|
+
isDirtyWhenAnalyzed: isDirty,
|
|
481
|
+
fileHashWhenAnalyzed: fileHash,
|
|
482
|
+
};
|
|
483
|
+
await this.storageService.saveFileAnalysis(this.projectSlug, relPath, fileResult);
|
|
484
|
+
completedFiles.push(relPath);
|
|
485
|
+
fileResults.set(relPath, fileResult);
|
|
486
|
+
params.onObjectCompleted?.(fileObj, { status: 'parsed' });
|
|
487
|
+
const sourceAbsPath = path.resolve(params.projectRoot, relPath);
|
|
488
|
+
const resultAbsPath = path.resolve(storageRoot, (0, utils_1.getFileOutputPath)(storageRoot, relPath));
|
|
489
|
+
indexEntries.push({ sourcePath: sourceAbsPath, resultPath: resultAbsPath, type: 'file' });
|
|
490
|
+
}
|
|
491
|
+
catch (e) {
|
|
492
|
+
errors.push({ path: relPath, message: e.message });
|
|
493
|
+
params.onObjectCompleted?.(fileObj, { status: 'failed', reason: e.message });
|
|
494
|
+
}
|
|
495
|
+
});
|
|
496
|
+
// --- 5b:目录聚合(叶子优先,按深度分组) ---
|
|
497
|
+
const dirsByDepth = new Map();
|
|
498
|
+
for (const dirRel of plannedDirs) {
|
|
499
|
+
const node = dirNodes.get(dirRel);
|
|
500
|
+
if (!node)
|
|
501
|
+
continue;
|
|
502
|
+
const arr = dirsByDepth.get(node.depth) ?? [];
|
|
503
|
+
arr.push(dirRel);
|
|
504
|
+
dirsByDepth.set(node.depth, arr);
|
|
505
|
+
}
|
|
506
|
+
const sortedDepths = Array.from(dirsByDepth.keys()).sort((a, b) => b - a);
|
|
507
|
+
for (const depth of sortedDepths) {
|
|
508
|
+
const batch = dirsByDepth.get(depth);
|
|
509
|
+
await (0, utils_1.mapLimit)(batch, params.concurrency, async (dirRel) => {
|
|
510
|
+
const dirObj = { type: 'directory', path: dirRel };
|
|
511
|
+
const node = dirNodes.get(dirRel);
|
|
512
|
+
if (!node) {
|
|
513
|
+
params.onObjectCompleted?.(dirObj, { status: 'skipped', reason: 'dir node not found' });
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
// 收集子项结果:优先从内存 Map 读取(刚解析的),回退到存储层(未变更的)
|
|
517
|
+
const childResults = [];
|
|
518
|
+
for (const f of node.childFiles) {
|
|
519
|
+
const fr = fileResults.get(f);
|
|
520
|
+
if (fr) {
|
|
521
|
+
childResults.push(fr);
|
|
522
|
+
}
|
|
523
|
+
else {
|
|
524
|
+
const stored = await this.storageService.getFileAnalysis(this.projectSlug, f, 'summary');
|
|
525
|
+
if (stored)
|
|
526
|
+
childResults.push(stored);
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
for (const d of node.childDirs) {
|
|
530
|
+
const dr = dirResults.get(d);
|
|
531
|
+
if (dr) {
|
|
532
|
+
childResults.push(dr);
|
|
533
|
+
}
|
|
534
|
+
else {
|
|
535
|
+
const stored = await this.storageService.getDirectoryAnalysis(this.projectSlug, d, 'summary');
|
|
536
|
+
if (stored)
|
|
537
|
+
childResults.push(stored);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
const fileChildren = childResults.filter(c => c.type === 'file');
|
|
541
|
+
const dirChildren = childResults.filter(c => c.type === 'directory');
|
|
542
|
+
if (fileChildren.length === 0 && dirChildren.length === 0) {
|
|
543
|
+
const out = (0, utils_1.getDirOutputPath)(storageRoot, dirRel);
|
|
544
|
+
if (await fs.pathExists(out)) {
|
|
545
|
+
await fs.remove(out);
|
|
546
|
+
}
|
|
547
|
+
params.onObjectCompleted?.(dirObj, { status: 'skipped', reason: 'empty directory' });
|
|
548
|
+
return;
|
|
549
|
+
}
|
|
550
|
+
const childrenDirsPayload = dirChildren.map(d => ({
|
|
551
|
+
name: d.name,
|
|
552
|
+
summary: d.summary,
|
|
553
|
+
description: d.description,
|
|
554
|
+
}));
|
|
555
|
+
const childrenFilesPayload = fileChildren.map(f => ({
|
|
556
|
+
name: f.name,
|
|
557
|
+
summary: f.summary,
|
|
558
|
+
description: f.description ?? f.summary,
|
|
559
|
+
}));
|
|
560
|
+
params.onObjectStarted?.(dirObj);
|
|
561
|
+
let description = '';
|
|
562
|
+
let summary = '';
|
|
563
|
+
try {
|
|
564
|
+
const llmRes = await workerPool.submitDirectoryAggregationTask(node.absPath, {
|
|
565
|
+
childrenDirs: childrenDirsPayload,
|
|
566
|
+
childrenFiles: childrenFilesPayload,
|
|
567
|
+
});
|
|
568
|
+
if (llmRes?.usage) {
|
|
569
|
+
this.tracker.addTotals(llmRes.usage);
|
|
570
|
+
}
|
|
571
|
+
description = llmRes.description;
|
|
572
|
+
summary = llmRes.summary;
|
|
573
|
+
}
|
|
574
|
+
catch (e) {
|
|
575
|
+
const dirName = path.basename(dirRel);
|
|
576
|
+
const fileCount = fileChildren.length;
|
|
577
|
+
const dirCount = dirChildren.length;
|
|
578
|
+
const fallback = `The "${dirName}" directory contains ${fileCount} file(s) and ${dirCount} subdirectory(ies) and helps organize related source code and modules.`;
|
|
579
|
+
description = fallback;
|
|
580
|
+
summary = fallback;
|
|
581
|
+
}
|
|
582
|
+
const dirResult = {
|
|
583
|
+
type: 'directory',
|
|
584
|
+
path: dirRel,
|
|
585
|
+
name: path.basename(dirRel),
|
|
586
|
+
description,
|
|
587
|
+
summary,
|
|
588
|
+
childrenDirsCount: dirChildren.length,
|
|
589
|
+
childrenFilesCount: fileChildren.length,
|
|
590
|
+
structure: childResults.map(child => ({
|
|
591
|
+
name: child.name,
|
|
592
|
+
type: child.type,
|
|
593
|
+
description: child.summary,
|
|
594
|
+
})),
|
|
595
|
+
lastAnalyzedAt: new Date().toISOString(),
|
|
596
|
+
commitHash: params.commitHash,
|
|
597
|
+
};
|
|
598
|
+
try {
|
|
599
|
+
await this.storageService.saveDirectoryAnalysis(this.projectSlug, dirRel, dirResult);
|
|
600
|
+
dirResults.set(dirRel, dirResult);
|
|
601
|
+
completedDirs.push(dirRel);
|
|
602
|
+
const dirSourceAbsPath = path.resolve(params.projectRoot, dirRel);
|
|
603
|
+
const dirResultAbsPath = path.resolve(storageRoot, (0, utils_1.getDirOutputPath)(storageRoot, dirRel));
|
|
604
|
+
indexEntries.push({ sourcePath: dirSourceAbsPath, resultPath: dirResultAbsPath, type: 'directory' });
|
|
605
|
+
params.onObjectCompleted?.(dirObj, { status: 'parsed' });
|
|
606
|
+
}
|
|
607
|
+
catch (e) {
|
|
608
|
+
const msg = e?.message ?? String(e);
|
|
609
|
+
errors.push({ path: dirRel, message: msg });
|
|
610
|
+
params.onObjectCompleted?.(dirObj, { status: 'failed', reason: msg });
|
|
611
|
+
return;
|
|
612
|
+
}
|
|
613
|
+
// 内存回收:子项结果在上层目录仅需 summary/description,可安全释放
|
|
614
|
+
for (const f of node.childFiles) {
|
|
615
|
+
fileResults.delete(f);
|
|
616
|
+
}
|
|
617
|
+
for (const d of node.childDirs) {
|
|
618
|
+
dirResults.delete(d);
|
|
619
|
+
}
|
|
620
|
+
});
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
catch (e) {
|
|
624
|
+
if (typeof workerPool.terminate === 'function') {
|
|
625
|
+
await workerPool.terminate(true).catch(() => { });
|
|
626
|
+
}
|
|
627
|
+
else {
|
|
628
|
+
workerPool.cancelAll();
|
|
629
|
+
}
|
|
630
|
+
throw e;
|
|
631
|
+
}
|
|
632
|
+
finally {
|
|
633
|
+
if (typeof workerPool.terminate === 'function') {
|
|
634
|
+
await workerPool.terminate(true).catch(() => { });
|
|
635
|
+
}
|
|
636
|
+
else {
|
|
637
|
+
workerPool.cancelAll();
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
const duration = Date.now() - startTime;
|
|
641
|
+
const summaryPath = path.join(storageRoot, 'index.md');
|
|
642
|
+
return {
|
|
643
|
+
success: errors.length === 0,
|
|
644
|
+
analyzedFilesCount: completedFiles.length,
|
|
645
|
+
analyzedDirsCount: completedDirs.length,
|
|
646
|
+
duration,
|
|
647
|
+
errors,
|
|
648
|
+
projectSlug: this.projectSlug,
|
|
649
|
+
summaryPath,
|
|
650
|
+
indexEntries,
|
|
651
|
+
removedSourcePaths,
|
|
652
|
+
};
|
|
653
|
+
}
|
|
654
|
+
// ---------------------------------------------------------------------------
|
|
655
|
+
// 向后兼容包装
|
|
656
|
+
// ---------------------------------------------------------------------------
|
|
657
|
+
async fullAnalysis(params) {
|
|
658
|
+
return this.analyze({
|
|
659
|
+
projectRoot: params.projectRoot,
|
|
660
|
+
depth: params.depth,
|
|
661
|
+
concurrency: params.concurrency,
|
|
662
|
+
mode: 'full',
|
|
663
|
+
commitHash: this.currentCommit,
|
|
664
|
+
fileFilter: async () => true,
|
|
665
|
+
onObjectPlanned: params.onObjectPlanned,
|
|
666
|
+
onObjectStarted: params.onObjectStarted,
|
|
667
|
+
onObjectCompleted: params.onObjectCompleted,
|
|
668
|
+
onScanProgress: params.onScanProgress,
|
|
669
|
+
});
|
|
670
|
+
}
|
|
671
|
+
async incrementalAnalysis(params) {
|
|
672
|
+
const changedFilesSet = new Set(params.changedFiles || []);
|
|
673
|
+
const storageRoot = this.storageService.getStoragePath(this.projectSlug);
|
|
674
|
+
const fileFilter = async (relPath, _absPath) => {
|
|
675
|
+
if (changedFilesSet.has(relPath))
|
|
676
|
+
return true;
|
|
677
|
+
const resultPath = (0, utils_1.getFileOutputPath)(storageRoot, relPath);
|
|
678
|
+
return !(await fs.pathExists(resultPath));
|
|
679
|
+
};
|
|
680
|
+
return this.analyze({
|
|
681
|
+
projectRoot: params.projectRoot,
|
|
682
|
+
concurrency: params.concurrency,
|
|
683
|
+
mode: 'incremental',
|
|
684
|
+
commitHash: params.targetCommit,
|
|
685
|
+
fileFilter,
|
|
686
|
+
onObjectPlanned: params.onObjectPlanned,
|
|
687
|
+
onObjectStarted: params.onObjectStarted,
|
|
688
|
+
onObjectCompleted: params.onObjectCompleted,
|
|
689
|
+
onScanProgress: params.onScanProgress,
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
async resumeAnalysis(params) {
|
|
693
|
+
throw new errors_1.AppError(errors_1.ErrorCode.ANALYSIS_EXCEPTION, 'Resume/checkpoint feature is not implemented yet');
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
exports.AnalysisService = AnalysisService;
|