veryfront 0.1.215 → 0.1.217
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/esm/cli/commands/extension/command-help.d.ts +3 -0
- package/esm/cli/commands/extension/command-help.d.ts.map +1 -0
- package/esm/cli/commands/extension/command-help.js +16 -0
- package/esm/cli/commands/extension/handler.d.ts +3 -0
- package/esm/cli/commands/extension/handler.d.ts.map +1 -0
- package/esm/cli/commands/extension/handler.js +36 -0
- package/esm/cli/commands/extension/init-command.d.ts +19 -0
- package/esm/cli/commands/extension/init-command.d.ts.map +1 -0
- package/esm/cli/commands/extension/init-command.js +122 -0
- package/esm/cli/commands/extension/validate-command.d.ts +13 -0
- package/esm/cli/commands/extension/validate-command.d.ts.map +1 -0
- package/esm/cli/commands/extension/validate-command.js +62 -0
- package/esm/cli/commands/generate/integration-generator-helpers.d.ts +35 -0
- package/esm/cli/commands/generate/integration-generator-helpers.d.ts.map +1 -0
- package/esm/cli/commands/generate/integration-generator-helpers.js +123 -0
- package/esm/cli/commands/generate/integration-generator.d.ts +1 -2
- package/esm/cli/commands/generate/integration-generator.d.ts.map +1 -1
- package/esm/cli/commands/generate/integration-generator.js +1 -123
- package/esm/cli/commands/knowledge/command-helpers.d.ts +48 -0
- package/esm/cli/commands/knowledge/command-helpers.d.ts.map +1 -0
- package/esm/cli/commands/knowledge/command-helpers.js +216 -0
- package/esm/cli/commands/knowledge/command.d.ts +12 -17
- package/esm/cli/commands/knowledge/command.d.ts.map +1 -1
- package/esm/cli/commands/knowledge/command.js +26 -214
- package/esm/cli/help/command-definitions.d.ts.map +1 -1
- package/esm/cli/help/command-definitions.js +2 -0
- package/esm/cli/mcp/remote-file-tool-helpers.d.ts +5 -0
- package/esm/cli/mcp/remote-file-tool-helpers.d.ts.map +1 -0
- package/esm/cli/mcp/remote-file-tool-helpers.js +22 -0
- package/esm/cli/mcp/remote-file-tools.d.ts.map +1 -1
- package/esm/cli/mcp/remote-file-tools.js +1 -22
- package/esm/cli/router.d.ts.map +1 -1
- package/esm/cli/router.js +2 -0
- package/esm/cli/templates/integration-loader-helpers.d.ts +8 -0
- package/esm/cli/templates/integration-loader-helpers.d.ts.map +1 -0
- package/esm/cli/templates/integration-loader-helpers.js +29 -0
- package/esm/cli/templates/integration-loader.d.ts.map +1 -1
- package/esm/cli/templates/integration-loader.js +5 -21
- package/esm/cli/templates/manifest.d.ts +1 -0
- package/esm/cli/templates/manifest.js +2 -1
- package/esm/deno.d.ts +3 -0
- package/esm/deno.js +5 -2
- package/esm/src/agent/runtime/index.d.ts.map +1 -1
- package/esm/src/agent/runtime/index.js +48 -90
- package/esm/src/chat/ag-ui-helpers.d.ts +17 -0
- package/esm/src/chat/ag-ui-helpers.d.ts.map +1 -0
- package/esm/src/chat/ag-ui-helpers.js +111 -0
- package/esm/src/chat/ag-ui.d.ts.map +1 -1
- package/esm/src/chat/ag-ui.js +1 -111
- package/esm/src/config/schemas/config.schema.d.ts +1 -0
- package/esm/src/config/schemas/config.schema.d.ts.map +1 -1
- package/esm/src/config/schemas/config.schema.js +11 -0
- package/esm/src/config/schemas/index.d.ts +11 -1
- package/esm/src/config/schemas/index.d.ts.map +1 -1
- package/esm/src/errors/error-registry-helpers.d.ts +6 -0
- package/esm/src/errors/error-registry-helpers.d.ts.map +1 -0
- package/esm/src/errors/error-registry-helpers.js +9 -0
- package/esm/src/errors/error-registry.d.ts.map +1 -1
- package/esm/src/errors/error-registry.js +4 -3
- package/esm/src/extensions/capabilities.d.ts +20 -0
- package/esm/src/extensions/capabilities.d.ts.map +1 -0
- package/esm/src/extensions/capabilities.js +77 -0
- package/esm/src/extensions/contracts.d.ts +10 -0
- package/esm/src/extensions/contracts.d.ts.map +1 -0
- package/esm/src/extensions/contracts.js +28 -0
- package/esm/src/extensions/discovery.d.ts +48 -0
- package/esm/src/extensions/discovery.d.ts.map +1 -0
- package/esm/src/extensions/discovery.js +179 -0
- package/esm/src/extensions/errors.d.ts +10 -0
- package/esm/src/extensions/errors.d.ts.map +1 -0
- package/esm/src/extensions/errors.js +34 -0
- package/esm/src/extensions/factory-loader.d.ts +29 -0
- package/esm/src/extensions/factory-loader.d.ts.map +1 -0
- package/esm/src/extensions/factory-loader.js +63 -0
- package/esm/src/extensions/index.d.ts +37 -0
- package/esm/src/extensions/index.d.ts.map +1 -0
- package/esm/src/extensions/index.js +41 -0
- package/esm/src/extensions/interfaces/ai-model-provider.d.ts +94 -0
- package/esm/src/extensions/interfaces/ai-model-provider.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/ai-model-provider.js +8 -0
- package/esm/src/extensions/interfaces/auth-provider.d.ts +49 -0
- package/esm/src/extensions/interfaces/auth-provider.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/auth-provider.js +8 -0
- package/esm/src/extensions/interfaces/bundler.d.ts +118 -0
- package/esm/src/extensions/interfaces/bundler.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/bundler.js +8 -0
- package/esm/src/extensions/interfaces/cache-store.d.ts +27 -0
- package/esm/src/extensions/interfaces/cache-store.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/cache-store.js +8 -0
- package/esm/src/extensions/interfaces/code-parser.d.ts +76 -0
- package/esm/src/extensions/interfaces/code-parser.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/code-parser.js +8 -0
- package/esm/src/extensions/interfaces/content-transformer.d.ts +40 -0
- package/esm/src/extensions/interfaces/content-transformer.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/content-transformer.js +8 -0
- package/esm/src/extensions/interfaces/css-processor.d.ts +38 -0
- package/esm/src/extensions/interfaces/css-processor.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/css-processor.js +8 -0
- package/esm/src/extensions/interfaces/database-client.d.ts +31 -0
- package/esm/src/extensions/interfaces/database-client.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/database-client.js +8 -0
- package/esm/src/extensions/interfaces/embedding-provider.d.ts +37 -0
- package/esm/src/extensions/interfaces/embedding-provider.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/embedding-provider.js +8 -0
- package/esm/src/extensions/interfaces/index.d.ts +21 -0
- package/esm/src/extensions/interfaces/index.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/index.js +9 -0
- package/esm/src/extensions/interfaces/node-compat.d.ts +25 -0
- package/esm/src/extensions/interfaces/node-compat.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/node-compat.js +8 -0
- package/esm/src/extensions/interfaces/schema-validator.d.ts +46 -0
- package/esm/src/extensions/interfaces/schema-validator.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/schema-validator.js +8 -0
- package/esm/src/extensions/interfaces/tracing-exporter.d.ts +44 -0
- package/esm/src/extensions/interfaces/tracing-exporter.d.ts.map +1 -0
- package/esm/src/extensions/interfaces/tracing-exporter.js +8 -0
- package/esm/src/extensions/loader.d.ts +33 -0
- package/esm/src/extensions/loader.d.ts.map +1 -0
- package/esm/src/extensions/loader.js +209 -0
- package/esm/src/extensions/orchestrate.d.ts +54 -0
- package/esm/src/extensions/orchestrate.d.ts.map +1 -0
- package/esm/src/extensions/orchestrate.js +116 -0
- package/esm/src/extensions/recommendations.d.ts +7 -0
- package/esm/src/extensions/recommendations.d.ts.map +1 -0
- package/esm/src/extensions/recommendations.js +22 -0
- package/esm/src/extensions/types.d.ts +47 -0
- package/esm/src/extensions/types.d.ts.map +1 -0
- package/esm/src/extensions/types.js +6 -0
- package/esm/src/extensions/validation.d.ts +44 -0
- package/esm/src/extensions/validation.d.ts.map +1 -0
- package/esm/src/extensions/validation.js +115 -0
- package/esm/src/html/styles-builder/plugin-loader.d.ts.map +1 -1
- package/esm/src/html/styles-builder/plugin-loader.js +23 -0
- package/esm/src/html/styles-builder/tailwind-plugin-allowlist.d.ts +29 -0
- package/esm/src/html/styles-builder/tailwind-plugin-allowlist.d.ts.map +1 -0
- package/esm/src/html/styles-builder/tailwind-plugin-allowlist.js +48 -0
- package/esm/src/modules/react-loader/ssr-module-loader/loader-helpers.d.ts +15 -0
- package/esm/src/modules/react-loader/ssr-module-loader/loader-helpers.d.ts.map +1 -0
- package/esm/src/modules/react-loader/ssr-module-loader/loader-helpers.js +22 -0
- package/esm/src/modules/react-loader/ssr-module-loader/loader.d.ts +0 -2
- package/esm/src/modules/react-loader/ssr-module-loader/loader.d.ts.map +1 -1
- package/esm/src/modules/react-loader/ssr-module-loader/loader.js +4 -24
- package/esm/src/platform/adapters/fs/veryfront/adapter-helpers.d.ts +19 -0
- package/esm/src/platform/adapters/fs/veryfront/adapter-helpers.d.ts.map +1 -0
- package/esm/src/platform/adapters/fs/veryfront/adapter-helpers.js +26 -0
- package/esm/src/platform/adapters/fs/veryfront/adapter.d.ts.map +1 -1
- package/esm/src/platform/adapters/fs/veryfront/adapter.js +4 -20
- package/esm/src/platform/adapters/fs/veryfront/read-operations-helpers.d.ts +1 -0
- package/esm/src/platform/adapters/fs/veryfront/read-operations-helpers.d.ts.map +1 -1
- package/esm/src/platform/adapters/fs/veryfront/read-operations-helpers.js +3 -0
- package/esm/src/platform/adapters/fs/veryfront/read-operations.d.ts.map +1 -1
- package/esm/src/platform/adapters/fs/veryfront/read-operations.js +4 -7
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager-helpers.d.ts +28 -0
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager-helpers.d.ts.map +1 -0
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager-helpers.js +41 -0
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager.d.ts +1 -4
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager.d.ts.map +1 -1
- package/esm/src/platform/adapters/fs/veryfront/websocket-manager.js +7 -40
- package/esm/src/platform/compat/process/command.d.ts +31 -0
- package/esm/src/platform/compat/process/command.d.ts.map +1 -0
- package/esm/src/platform/compat/process/command.js +193 -0
- package/esm/src/platform/compat/process/env.d.ts +31 -0
- package/esm/src/platform/compat/process/env.d.ts.map +1 -0
- package/esm/src/platform/compat/process/env.js +175 -0
- package/esm/src/platform/compat/process/lifecycle.d.ts +98 -0
- package/esm/src/platform/compat/process/lifecycle.d.ts.map +1 -0
- package/esm/src/platform/compat/process/lifecycle.js +305 -0
- package/esm/src/platform/compat/process/runtime-process.d.ts +10 -0
- package/esm/src/platform/compat/process/runtime-process.d.ts.map +1 -0
- package/esm/src/platform/compat/process/runtime-process.js +24 -0
- package/esm/src/platform/compat/process.d.ts +4 -165
- package/esm/src/platform/compat/process.d.ts.map +1 -1
- package/esm/src/platform/compat/process.js +4 -682
- package/esm/src/platform/index.d.ts +1 -1
- package/esm/src/platform/index.d.ts.map +1 -1
- package/esm/src/provider/runtime-loader/provider-endpoints.d.ts +8 -0
- package/esm/src/provider/runtime-loader/provider-endpoints.d.ts.map +1 -0
- package/esm/src/provider/runtime-loader/provider-endpoints.js +27 -0
- package/esm/src/provider/runtime-loader/provider-request-init.d.ts +32 -0
- package/esm/src/provider/runtime-loader/provider-request-init.d.ts.map +1 -0
- package/esm/src/provider/runtime-loader/provider-request-init.js +67 -0
- package/esm/src/provider/runtime-loader/tool-input-status.d.ts +17 -0
- package/esm/src/provider/runtime-loader/tool-input-status.d.ts.map +1 -0
- package/esm/src/provider/runtime-loader/tool-input-status.js +155 -0
- package/esm/src/provider/runtime-loader.d.ts +2 -3
- package/esm/src/provider/runtime-loader.d.ts.map +1 -1
- package/esm/src/provider/runtime-loader.js +92 -291
- package/esm/src/proxy/handler.d.ts.map +1 -1
- package/esm/src/proxy/handler.js +14 -0
- package/esm/src/rendering/orchestrator/pipeline-helpers.d.ts +8 -0
- package/esm/src/rendering/orchestrator/pipeline-helpers.d.ts.map +1 -0
- package/esm/src/rendering/orchestrator/pipeline-helpers.js +20 -0
- package/esm/src/rendering/orchestrator/pipeline.d.ts +0 -3
- package/esm/src/rendering/orchestrator/pipeline.d.ts.map +1 -1
- package/esm/src/rendering/orchestrator/pipeline.js +4 -22
- package/esm/src/routing/api/module-loader/loader-helpers.d.ts +10 -0
- package/esm/src/routing/api/module-loader/loader-helpers.d.ts.map +1 -0
- package/esm/src/routing/api/module-loader/loader-helpers.js +62 -0
- package/esm/src/routing/api/module-loader/loader.d.ts +1 -1
- package/esm/src/routing/api/module-loader/loader.d.ts.map +1 -1
- package/esm/src/routing/api/module-loader/loader.js +2 -60
- package/esm/src/server/bootstrap.d.ts +22 -2
- package/esm/src/server/bootstrap.d.ts.map +1 -1
- package/esm/src/server/bootstrap.js +67 -5
- package/esm/src/server/dev-ui/manifest.d.ts +2 -0
- package/esm/src/server/dev-ui/manifest.js +3 -1
- package/esm/src/server/handlers/request/api/project-discovery.d.ts.map +1 -1
- package/esm/src/server/handlers/request/api/project-discovery.js +14 -8
- package/esm/src/server/production-server.js +1 -1
- package/esm/src/server/services/rsc/endpoints/rsc-bundles.generated.d.ts.map +1 -1
- package/esm/src/server/services/rsc/endpoints/rsc-bundles.generated.js +2 -2
- package/esm/src/server/utils/domain-parser.d.ts.map +1 -1
- package/esm/src/server/utils/domain-parser.js +4 -0
- package/esm/src/studio/bridge/bridge-bundle.generated.d.ts.map +1 -1
- package/esm/src/studio/bridge/bridge-bundle.generated.js +1 -1
- package/esm/src/tool/index.d.ts +1 -1
- package/esm/src/tool/index.d.ts.map +1 -1
- package/esm/src/tool/types.d.ts +20 -0
- package/esm/src/tool/types.d.ts.map +1 -1
- package/esm/src/utils/version-constant.d.ts +1 -1
- package/esm/src/utils/version-constant.js +1 -1
- package/package.json +7 -1
- package/src/cli/commands/extension/command-help.ts +18 -0
- package/src/cli/commands/extension/handler.ts +41 -0
- package/src/cli/commands/extension/init-command.ts +140 -0
- package/src/cli/commands/extension/validate-command.ts +78 -0
- package/src/cli/commands/generate/integration-generator-helpers.ts +185 -0
- package/src/cli/commands/generate/integration-generator.ts +12 -168
- package/src/cli/commands/knowledge/command-helpers.ts +295 -0
- package/src/cli/commands/knowledge/command.ts +34 -261
- package/src/cli/help/command-definitions.ts +2 -0
- package/src/cli/mcp/remote-file-tool-helpers.ts +27 -0
- package/src/cli/mcp/remote-file-tools.ts +6 -28
- package/src/cli/router.ts +2 -0
- package/src/cli/templates/integration-loader-helpers.ts +49 -0
- package/src/cli/templates/integration-loader.ts +10 -28
- package/src/cli/templates/manifest.js +2 -1
- package/src/deno.js +5 -2
- package/src/src/agent/runtime/index.ts +77 -94
- package/src/src/chat/ag-ui-helpers.ts +139 -0
- package/src/src/chat/ag-ui.ts +11 -139
- package/src/src/config/schemas/config.schema.ts +11 -0
- package/src/src/config/schemas/index.ts +15 -1
- package/src/src/errors/error-registry-helpers.ts +23 -0
- package/src/src/errors/error-registry.ts +8 -3
- package/src/src/extensions/capabilities.ts +97 -0
- package/src/src/extensions/contracts.ts +36 -0
- package/src/src/extensions/discovery.ts +221 -0
- package/src/src/extensions/errors.ts +39 -0
- package/src/src/extensions/factory-loader.ts +76 -0
- package/src/src/extensions/index.ts +79 -0
- package/src/src/extensions/interfaces/ai-model-provider.ts +100 -0
- package/src/src/extensions/interfaces/auth-provider.ts +52 -0
- package/src/src/extensions/interfaces/bundler.ts +116 -0
- package/src/src/extensions/interfaces/cache-store.ts +27 -0
- package/src/src/extensions/interfaces/code-parser.ts +84 -0
- package/src/src/extensions/interfaces/content-transformer.ts +38 -0
- package/src/src/extensions/interfaces/css-processor.ts +40 -0
- package/src/src/extensions/interfaces/database-client.ts +35 -0
- package/src/src/extensions/interfaces/embedding-provider.ts +39 -0
- package/src/src/extensions/interfaces/index.ts +81 -0
- package/src/src/extensions/interfaces/node-compat.ts +25 -0
- package/src/src/extensions/interfaces/schema-validator.ts +51 -0
- package/src/src/extensions/interfaces/tracing-exporter.ts +42 -0
- package/src/src/extensions/loader.ts +245 -0
- package/src/src/extensions/orchestrate.ts +184 -0
- package/src/src/extensions/recommendations.ts +24 -0
- package/src/src/extensions/types.ts +57 -0
- package/src/src/extensions/validation.ts +147 -0
- package/src/src/html/styles-builder/plugin-loader.ts +32 -0
- package/src/src/html/styles-builder/tailwind-plugin-allowlist.ts +51 -0
- package/src/src/modules/react-loader/ssr-module-loader/loader-helpers.ts +37 -0
- package/src/src/modules/react-loader/ssr-module-loader/loader.ts +8 -39
- package/src/src/platform/adapters/fs/veryfront/adapter-helpers.ts +43 -0
- package/src/src/platform/adapters/fs/veryfront/adapter.ts +8 -22
- package/src/src/platform/adapters/fs/veryfront/read-operations-helpers.ts +4 -0
- package/src/src/platform/adapters/fs/veryfront/read-operations.ts +4 -7
- package/src/src/platform/adapters/fs/veryfront/websocket-manager-helpers.ts +73 -0
- package/src/src/platform/adapters/fs/veryfront/websocket-manager.ts +29 -44
- package/src/src/platform/compat/process/command.ts +297 -0
- package/src/src/platform/compat/process/env.ts +227 -0
- package/src/src/platform/compat/process/lifecycle.ts +330 -0
- package/src/src/platform/compat/process/runtime-process.ts +27 -0
- package/src/src/platform/compat/process.ts +37 -870
- package/src/src/platform/index.ts +1 -0
- package/src/src/provider/runtime-loader/provider-endpoints.ts +50 -0
- package/src/src/provider/runtime-loader/provider-request-init.ts +101 -0
- package/src/src/provider/runtime-loader/tool-input-status.ts +210 -0
- package/src/src/provider/runtime-loader.ts +113 -368
- package/src/src/proxy/handler.ts +16 -0
- package/src/src/rendering/orchestrator/pipeline-helpers.ts +35 -0
- package/src/src/rendering/orchestrator/pipeline.ts +8 -35
- package/src/src/routing/api/module-loader/loader-helpers.ts +68 -0
- package/src/src/routing/api/module-loader/loader.ts +8 -65
- package/src/src/server/bootstrap.ts +88 -7
- package/src/src/server/dev-ui/manifest.js +3 -1
- package/src/src/server/handlers/request/api/project-discovery.ts +19 -8
- package/src/src/server/production-server.ts +1 -1
- package/src/src/server/services/rsc/endpoints/rsc-bundles.generated.ts +2 -2
- package/src/src/server/utils/domain-parser.ts +4 -0
- package/src/src/studio/bridge/bridge-bundle.generated.ts +1 -1
- package/src/src/tool/index.ts +1 -0
- package/src/src/tool/types.ts +21 -0
- package/src/src/utils/version-constant.ts +1 -1
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import { createFileSystem } from "../../../src/platform/index.js";
|
|
2
|
+
import { basename, extname, join, normalize, relative } from "../../../src/platform/compat/path/index.js";
|
|
3
|
+
import { classifyKnowledgeDirectoryPath, classifyKnowledgeSourcePath } from "./source-policy.js";
|
|
4
|
+
import type {
|
|
5
|
+
KnowledgeIngestFailedFileResult,
|
|
6
|
+
KnowledgeIngestFileResult,
|
|
7
|
+
KnowledgeIngestSkippedFileResult,
|
|
8
|
+
} from "./result.js";
|
|
9
|
+
|
|
10
|
+
export type KnowledgeSource =
|
|
11
|
+
| { kind: "local"; input: string; localPath: string }
|
|
12
|
+
| { kind: "upload"; input: string; uploadPath: string; localPath: string };
|
|
13
|
+
|
|
14
|
+
export interface KnowledgeSourceCollection {
|
|
15
|
+
sources: KnowledgeSource[];
|
|
16
|
+
skipped: KnowledgeIngestSkippedFileResult[];
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function slugify(value: string): string {
|
|
20
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "document";
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const CHAT_UPLOAD_PREFIX_RE =
|
|
24
|
+
/^chat-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-\d+-[a-z0-9]+-/i;
|
|
25
|
+
|
|
26
|
+
export function normalizeKnowledgeInputPath(inputPath: string): string {
|
|
27
|
+
const normalizedPath = normalize(inputPath).replace(/^\/+/, "").replace(/\\/g, "/");
|
|
28
|
+
if (!normalizedPath || normalizedPath.startsWith("..") || normalizedPath.startsWith("/")) {
|
|
29
|
+
throw new Error(`Invalid knowledge input path: ${inputPath}`);
|
|
30
|
+
}
|
|
31
|
+
return normalizedPath;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function normalizeProjectUploadPath(inputPath: string): string {
|
|
35
|
+
return normalizeKnowledgeInputPath(inputPath);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function formatKnowledgeUploadSource(uploadPath: string): string {
|
|
39
|
+
const normalizedPath = normalizeKnowledgeInputPath(uploadPath);
|
|
40
|
+
return normalizedPath === "uploads" || normalizedPath.startsWith("uploads/")
|
|
41
|
+
? normalizedPath
|
|
42
|
+
: `uploads/${normalizedPath}`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function resolveExplicitUploadPath(inputPath: string): string {
|
|
46
|
+
const normalizedInput = normalizeKnowledgeInputPath(inputPath);
|
|
47
|
+
const displayInput = inputPath.replace(/\\/g, "/");
|
|
48
|
+
const uploadPath = normalizeProjectUploadPath(inputPath);
|
|
49
|
+
if (
|
|
50
|
+
!uploadPath || uploadPath === "uploads" || normalizedInput === "uploads" ||
|
|
51
|
+
normalizedInput.endsWith("/")
|
|
52
|
+
) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`Directory upload references require --path <prefix> --all: ${displayInput}`,
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
return uploadPath;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function isLikelyLocalPath(value: string): boolean {
|
|
61
|
+
return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") ||
|
|
62
|
+
/^[A-Za-z]:[\\/]/.test(value);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function isProjectUploadReference(value: string): boolean {
|
|
66
|
+
if (isLikelyLocalPath(value)) return false;
|
|
67
|
+
const normalizedValue = normalize(value).replace(/\\/g, "/").replace(/^\/+/, "");
|
|
68
|
+
return normalizedValue === "uploads" || normalizedValue.startsWith("uploads/");
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function stripChatUploadPrefix(name: string): string {
|
|
72
|
+
return name.replace(CHAT_UPLOAD_PREFIX_RE, "");
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function resolveKnowledgeDownloadOutputDir(outputDir: string): string {
|
|
76
|
+
return join(outputDir, ".uploads");
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function createSkippedKnowledgeSource(input: {
|
|
80
|
+
source: string;
|
|
81
|
+
localSourcePath?: string | null;
|
|
82
|
+
message: string;
|
|
83
|
+
reason: KnowledgeIngestSkippedFileResult["reason"];
|
|
84
|
+
}): KnowledgeIngestSkippedFileResult {
|
|
85
|
+
return {
|
|
86
|
+
source: input.source,
|
|
87
|
+
localSourcePath: input.localSourcePath ?? null,
|
|
88
|
+
message: input.message,
|
|
89
|
+
reason: input.reason,
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function createFailedKnowledgeSource(input: {
|
|
94
|
+
source: string;
|
|
95
|
+
localSourcePath: string;
|
|
96
|
+
message: string;
|
|
97
|
+
reason: KnowledgeIngestFailedFileResult["reason"];
|
|
98
|
+
}): KnowledgeIngestFailedFileResult {
|
|
99
|
+
return {
|
|
100
|
+
source: input.source,
|
|
101
|
+
localSourcePath: input.localSourcePath,
|
|
102
|
+
message: input.message,
|
|
103
|
+
reason: input.reason,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function classifySourceOrSkip(input: {
|
|
108
|
+
source: string;
|
|
109
|
+
localSourcePath?: string | null;
|
|
110
|
+
}): KnowledgeIngestSkippedFileResult | null {
|
|
111
|
+
const decision = classifyKnowledgeSourcePath(input.source);
|
|
112
|
+
if (decision.kind === "ingest") {
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return createSkippedKnowledgeSource({
|
|
117
|
+
source: input.source,
|
|
118
|
+
localSourcePath: input.localSourcePath,
|
|
119
|
+
message: decision.message,
|
|
120
|
+
reason: decision.reason,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function classifyDirectoryOrSkip(
|
|
125
|
+
input: { source: string },
|
|
126
|
+
): KnowledgeIngestSkippedFileResult | null {
|
|
127
|
+
const decision = classifyKnowledgeDirectoryPath(input.source);
|
|
128
|
+
if (decision.kind === "ingest") {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return createSkippedKnowledgeSource({
|
|
133
|
+
source: input.source,
|
|
134
|
+
localSourcePath: null,
|
|
135
|
+
message: decision.message,
|
|
136
|
+
reason: decision.reason,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export async function collectLocalFiles(
|
|
141
|
+
root: string,
|
|
142
|
+
recursive: boolean,
|
|
143
|
+
): Promise<KnowledgeSourceCollection> {
|
|
144
|
+
const fs = createFileSystem();
|
|
145
|
+
const stat = await fs.stat(root);
|
|
146
|
+
if (stat.isFile) {
|
|
147
|
+
const skipped = classifySourceOrSkip({ source: root, localSourcePath: root });
|
|
148
|
+
return skipped == null
|
|
149
|
+
? {
|
|
150
|
+
sources: [{ kind: "local", input: root, localPath: root }],
|
|
151
|
+
skipped: [],
|
|
152
|
+
}
|
|
153
|
+
: {
|
|
154
|
+
sources: [],
|
|
155
|
+
skipped: [skipped],
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
if (!stat.isDirectory) {
|
|
159
|
+
return { sources: [], skipped: [] };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const skippedRootDirectory = classifyDirectoryOrSkip({ source: root });
|
|
163
|
+
if (skippedRootDirectory != null) {
|
|
164
|
+
return {
|
|
165
|
+
sources: [],
|
|
166
|
+
skipped: [skippedRootDirectory],
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const collection: KnowledgeSourceCollection = {
|
|
171
|
+
sources: [],
|
|
172
|
+
skipped: [],
|
|
173
|
+
};
|
|
174
|
+
async function walk(dir: string): Promise<void> {
|
|
175
|
+
for await (const entry of fs.readDir(dir)) {
|
|
176
|
+
const entryPath = join(dir, entry.name);
|
|
177
|
+
if (entry.isDirectory) {
|
|
178
|
+
const skipped = classifyDirectoryOrSkip({ source: entryPath });
|
|
179
|
+
if (skipped != null) {
|
|
180
|
+
collection.skipped.push(skipped);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (recursive) await walk(entryPath);
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (!entry.isFile) {
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const skipped = classifySourceOrSkip({ source: entryPath, localSourcePath: entryPath });
|
|
192
|
+
if (skipped != null) {
|
|
193
|
+
collection.skipped.push(skipped);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
collection.sources.push({ kind: "local", input: root, localPath: entryPath });
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
await walk(root);
|
|
202
|
+
collection.sources.sort((left, right) => left.localPath.localeCompare(right.localPath));
|
|
203
|
+
collection.skipped.sort((left, right) => left.source.localeCompare(right.source));
|
|
204
|
+
return collection;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
export function buildSourceReference(source: KnowledgeSource): string {
|
|
208
|
+
return source.kind === "upload"
|
|
209
|
+
? formatKnowledgeUploadSource(source.uploadPath)
|
|
210
|
+
: source.localPath;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
export function buildSuggestedSlug(source: KnowledgeSource, index: number): string {
|
|
214
|
+
const normalized = normalize(
|
|
215
|
+
source.kind === "upload" ? source.uploadPath : source.localPath,
|
|
216
|
+
).replace(/\\/g, "/");
|
|
217
|
+
|
|
218
|
+
let stripped: string;
|
|
219
|
+
if (source.kind === "upload") {
|
|
220
|
+
stripped = normalized
|
|
221
|
+
.replace(/^\/workspace\/uploads\//, "")
|
|
222
|
+
.replace(/^\/workspace\//, "")
|
|
223
|
+
.replace(/^uploads\//, "")
|
|
224
|
+
.replace(/\.[^.]+$/, "");
|
|
225
|
+
} else if (normalized.startsWith("/workspace/uploads/")) {
|
|
226
|
+
stripped = normalized.replace(/^\/workspace\/uploads\//, "").replace(/\.[^.]+$/, "");
|
|
227
|
+
} else if (normalized.startsWith("/workspace/")) {
|
|
228
|
+
stripped = normalized.replace(/^\/workspace\//, "").replace(/\.[^.]+$/, "");
|
|
229
|
+
} else if (normalized.startsWith("/")) {
|
|
230
|
+
stripped = basename(normalized, extname(normalized));
|
|
231
|
+
} else {
|
|
232
|
+
stripped = normalized.replace(/\.[^.]+$/, "");
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const lastSlash = stripped.lastIndexOf("/");
|
|
236
|
+
if (lastSlash >= 0) {
|
|
237
|
+
const dir = stripped.slice(0, lastSlash + 1);
|
|
238
|
+
const file = stripChatUploadPrefix(stripped.slice(lastSlash + 1));
|
|
239
|
+
stripped = file ? `${dir}${file}` : stripped;
|
|
240
|
+
} else {
|
|
241
|
+
stripped = stripChatUploadPrefix(stripped) || stripped;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
return slugify(stripped || basename(normalized, extname(normalized)) || `document-${index + 1}`);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export function ensureUniqueSlugs(sources: KnowledgeSource[]): string[] {
|
|
248
|
+
const counts = new Map<string, number>();
|
|
249
|
+
return sources.map((source, index) => {
|
|
250
|
+
const baseSlug = buildSuggestedSlug(source, index);
|
|
251
|
+
const nextCount = (counts.get(baseSlug) ?? 0) + 1;
|
|
252
|
+
counts.set(baseSlug, nextCount);
|
|
253
|
+
return nextCount === 1 ? baseSlug : `${baseSlug}-${nextCount}`;
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
export function deriveKnowledgeRemotePath(
|
|
258
|
+
outputPath: string,
|
|
259
|
+
outputDir: string,
|
|
260
|
+
knowledgePath: string,
|
|
261
|
+
): string {
|
|
262
|
+
const relativeOutputPath = relative(outputDir, outputPath).replace(/\\/g, "/");
|
|
263
|
+
if (!relativeOutputPath || relativeOutputPath.startsWith("..")) {
|
|
264
|
+
throw new Error(`Output path is outside output directory: ${outputPath}`);
|
|
265
|
+
}
|
|
266
|
+
const prefix = normalizeKnowledgeInputPath(knowledgePath);
|
|
267
|
+
const normalizedRelative = normalize(relativeOutputPath).replace(/^\/+/, "");
|
|
268
|
+
return `${prefix}/${normalizedRelative}`.replace(/\\/g, "/");
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function createKnowledgeIngestResult(input: {
|
|
272
|
+
source: string;
|
|
273
|
+
localSourcePath: string;
|
|
274
|
+
outputPath: string;
|
|
275
|
+
remotePath: string;
|
|
276
|
+
parser: {
|
|
277
|
+
slug: string;
|
|
278
|
+
stats: Record<string, unknown>;
|
|
279
|
+
warnings: string[];
|
|
280
|
+
source_type: string;
|
|
281
|
+
summary: string;
|
|
282
|
+
};
|
|
283
|
+
}): KnowledgeIngestFileResult {
|
|
284
|
+
return {
|
|
285
|
+
source: input.source,
|
|
286
|
+
localSourcePath: input.localSourcePath,
|
|
287
|
+
outputPath: input.outputPath,
|
|
288
|
+
remotePath: input.remotePath,
|
|
289
|
+
slug: input.parser.slug,
|
|
290
|
+
sourceType: input.parser.source_type,
|
|
291
|
+
summary: input.parser.summary,
|
|
292
|
+
stats: input.parser.stats,
|
|
293
|
+
warnings: input.parser.warnings,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import * as dntShim from "../../../_dnt.shims.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
type SafeParseResult<T> = { success: true; data: T } | { success: false; error: z.ZodError };
|
|
4
|
-
import { createFileSystem, getEnv } from "../../../src/platform/index.js";
|
|
5
|
-
import { basename
|
|
6
|
-
import { type CommandResult, runCommand } from "../../../src/platform/compat/process.js";
|
|
4
|
+
import { type CommandResult, createFileSystem, getEnv, runCommand } from "../../../src/platform/index.js";
|
|
5
|
+
import { basename } from "../../../src/platform/compat/path/index.js";
|
|
7
6
|
import { withSpan } from "../../../src/observability/tracing/otlp-setup.js";
|
|
8
7
|
import { cliLogger } from "../../utils/index.js";
|
|
9
8
|
import { type ApiClient, createApiClient, resolveConfigWithAuth } from "../../shared/config.js";
|
|
@@ -12,9 +11,10 @@ import { getStringArg } from "../../shared/parsed-args.js";
|
|
|
12
11
|
import { downloadUploadToFile, listAllUploads, type UploadItem } from "../uploads/command.js";
|
|
13
12
|
import { putRemoteFileFromLocal } from "../files/command.js";
|
|
14
13
|
import { knowledgeIngestPythonSource } from "./parser-source.js";
|
|
14
|
+
import * as commandHelpers from "./command-helpers.js";
|
|
15
15
|
import { createJobUserLogger, type Logger, serverLogger } from "../../../src/utils/index.js";
|
|
16
16
|
import { writeJobResultIfConfigured } from "../../utils/write-job-result.js";
|
|
17
|
-
import {
|
|
17
|
+
import { classifyKnowledgeSourcePath } from "./source-policy.js";
|
|
18
18
|
import {
|
|
19
19
|
buildKnowledgeIngestJobResult,
|
|
20
20
|
type KnowledgeIngestFailedFileResult,
|
|
@@ -173,108 +173,10 @@ export function parseKnowledgeIngestArgs(
|
|
|
173
173
|
});
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
export function normalizeKnowledgeInputPath(inputPath: string): string {
|
|
177
|
-
const normalizedPath = normalize(inputPath).replace(/^\/+/, "").replace(/\\/g, "/");
|
|
178
|
-
if (!normalizedPath || normalizedPath.startsWith("..") || normalizedPath.startsWith("/")) {
|
|
179
|
-
throw new Error(`Invalid knowledge input path: ${inputPath}`);
|
|
180
|
-
}
|
|
181
|
-
return normalizedPath;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
export function normalizeProjectUploadPath(inputPath: string): string {
|
|
185
|
-
return normalizeKnowledgeInputPath(inputPath);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
export function formatKnowledgeUploadSource(uploadPath: string): string {
|
|
189
|
-
const normalizedPath = normalizeKnowledgeInputPath(uploadPath);
|
|
190
|
-
return normalizedPath === "uploads" || normalizedPath.startsWith("uploads/")
|
|
191
|
-
? normalizedPath
|
|
192
|
-
: `uploads/${normalizedPath}`;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
function resolveExplicitUploadPath(inputPath: string): string {
|
|
196
|
-
const normalizedInput = normalizeKnowledgeInputPath(inputPath);
|
|
197
|
-
const displayInput = inputPath.replace(/\\/g, "/");
|
|
198
|
-
const uploadPath = normalizeProjectUploadPath(inputPath);
|
|
199
|
-
if (
|
|
200
|
-
!uploadPath || uploadPath === "uploads" || normalizedInput === "uploads" ||
|
|
201
|
-
normalizedInput.endsWith("/")
|
|
202
|
-
) {
|
|
203
|
-
throw new Error(
|
|
204
|
-
`Directory upload references require --path <prefix> --all: ${displayInput}`,
|
|
205
|
-
);
|
|
206
|
-
}
|
|
207
|
-
return uploadPath;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
export function isLikelyLocalPath(value: string): boolean {
|
|
211
|
-
return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") ||
|
|
212
|
-
/^[A-Za-z]:[\\/]/.test(value);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
function isProjectUploadReference(value: string): boolean {
|
|
216
|
-
if (isLikelyLocalPath(value)) return false;
|
|
217
|
-
const normalizedValue = normalize(value).replace(/\\/g, "/").replace(/^\/+/, "");
|
|
218
|
-
return normalizedValue === "uploads" || normalizedValue.startsWith("uploads/");
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
function slugify(value: string): string {
|
|
222
|
-
return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "document";
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/**
|
|
226
|
-
* Strip the chat-upload prefix that Studio prepends to uploaded files.
|
|
227
|
-
*
|
|
228
|
-
* Studio stores uploads with a generated prefix like:
|
|
229
|
-
* chat-<uuid>-<timestamp>-<shortid>-<original-filename>
|
|
230
|
-
*
|
|
231
|
-
* This function removes the prefix so knowledge files use the clean
|
|
232
|
-
* original filename (e.g. "agents" instead of
|
|
233
|
-
* "chat-909d3dbc-5a9a-4156-97e4-bcceb5c2ec0d-1773942180291-fv1qg5-agents").
|
|
234
|
-
*/
|
|
235
|
-
const CHAT_UPLOAD_PREFIX_RE =
|
|
236
|
-
/^chat-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-\d+-[a-z0-9]+-/i;
|
|
237
|
-
|
|
238
|
-
export function stripChatUploadPrefix(name: string): string {
|
|
239
|
-
return name.replace(CHAT_UPLOAD_PREFIX_RE, "");
|
|
240
|
-
}
|
|
241
|
-
|
|
242
176
|
function defaultOutputRoot(): Promise<string> {
|
|
243
177
|
return dntShim.Deno.makeTempDir({ prefix: "veryfront-knowledge-" });
|
|
244
178
|
}
|
|
245
179
|
|
|
246
|
-
export function resolveKnowledgeDownloadOutputDir(outputDir: string): string {
|
|
247
|
-
return join(outputDir, ".uploads");
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
function createSkippedKnowledgeSource(input: {
|
|
251
|
-
source: string;
|
|
252
|
-
localSourcePath?: string | null;
|
|
253
|
-
message: string;
|
|
254
|
-
reason: KnowledgeIngestSkippedFileResult["reason"];
|
|
255
|
-
}): KnowledgeIngestSkippedFileResult {
|
|
256
|
-
return {
|
|
257
|
-
source: input.source,
|
|
258
|
-
localSourcePath: input.localSourcePath ?? null,
|
|
259
|
-
message: input.message,
|
|
260
|
-
reason: input.reason,
|
|
261
|
-
};
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
function createFailedKnowledgeSource(input: {
|
|
265
|
-
source: string;
|
|
266
|
-
localSourcePath: string;
|
|
267
|
-
message: string;
|
|
268
|
-
reason: KnowledgeIngestFailedFileResult["reason"];
|
|
269
|
-
}): KnowledgeIngestFailedFileResult {
|
|
270
|
-
return {
|
|
271
|
-
source: input.source,
|
|
272
|
-
localSourcePath: input.localSourcePath,
|
|
273
|
-
message: input.message,
|
|
274
|
-
reason: input.reason,
|
|
275
|
-
};
|
|
276
|
-
}
|
|
277
|
-
|
|
278
180
|
function classifySourceOrSkip(input: {
|
|
279
181
|
source: string;
|
|
280
182
|
localSourcePath?: string | null;
|
|
@@ -284,180 +186,51 @@ function classifySourceOrSkip(input: {
|
|
|
284
186
|
return null;
|
|
285
187
|
}
|
|
286
188
|
|
|
287
|
-
return
|
|
189
|
+
return {
|
|
288
190
|
source: input.source,
|
|
289
|
-
localSourcePath: input.localSourcePath,
|
|
191
|
+
localSourcePath: input.localSourcePath ?? null,
|
|
290
192
|
message: decision.message,
|
|
291
193
|
reason: decision.reason,
|
|
292
|
-
}
|
|
194
|
+
};
|
|
293
195
|
}
|
|
294
196
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
197
|
+
export const normalizeKnowledgeInputPath = commandHelpers.normalizeKnowledgeInputPath;
|
|
198
|
+
export const normalizeProjectUploadPath = commandHelpers.normalizeProjectUploadPath;
|
|
199
|
+
export const formatKnowledgeUploadSource = commandHelpers.formatKnowledgeUploadSource;
|
|
200
|
+
export const isLikelyLocalPath = commandHelpers.isLikelyLocalPath;
|
|
201
|
+
export const stripChatUploadPrefix = commandHelpers.stripChatUploadPrefix;
|
|
202
|
+
export const resolveKnowledgeDownloadOutputDir = commandHelpers.resolveKnowledgeDownloadOutputDir;
|
|
203
|
+
export const buildSuggestedSlug = commandHelpers.buildSuggestedSlug;
|
|
204
|
+
export const ensureUniqueSlugs = commandHelpers.ensureUniqueSlugs;
|
|
205
|
+
export const deriveKnowledgeRemotePath = commandHelpers.deriveKnowledgeRemotePath;
|
|
206
|
+
export const createKnowledgeIngestResult = commandHelpers.createKnowledgeIngestResult;
|
|
302
207
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
208
|
+
function resolveExplicitUploadPath(inputPath: string): string {
|
|
209
|
+
return commandHelpers.resolveExplicitUploadPath(inputPath);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function isProjectUploadReference(value: string): boolean {
|
|
213
|
+
return commandHelpers.isProjectUploadReference(value);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function createFailedKnowledgeSource(input: {
|
|
217
|
+
source: string;
|
|
218
|
+
localSourcePath: string;
|
|
219
|
+
message: string;
|
|
220
|
+
reason: KnowledgeIngestFailedFileResult["reason"];
|
|
221
|
+
}): KnowledgeIngestFailedFileResult {
|
|
222
|
+
return commandHelpers.createFailedKnowledgeSource(input);
|
|
309
223
|
}
|
|
310
224
|
|
|
311
225
|
async function collectLocalFiles(
|
|
312
226
|
root: string,
|
|
313
227
|
recursive: boolean,
|
|
314
228
|
): Promise<KnowledgeSourceCollection> {
|
|
315
|
-
|
|
316
|
-
const stat = await fs.stat(root);
|
|
317
|
-
if (stat.isFile) {
|
|
318
|
-
const skipped = classifySourceOrSkip({ source: root, localSourcePath: root });
|
|
319
|
-
return skipped == null
|
|
320
|
-
? {
|
|
321
|
-
sources: [{ kind: "local", input: root, localPath: root }],
|
|
322
|
-
skipped: [],
|
|
323
|
-
}
|
|
324
|
-
: {
|
|
325
|
-
sources: [],
|
|
326
|
-
skipped: [skipped],
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
if (!stat.isDirectory) {
|
|
330
|
-
return { sources: [], skipped: [] };
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
const skippedRootDirectory = classifyDirectoryOrSkip({ source: root });
|
|
334
|
-
if (skippedRootDirectory != null) {
|
|
335
|
-
return {
|
|
336
|
-
sources: [],
|
|
337
|
-
skipped: [skippedRootDirectory],
|
|
338
|
-
};
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
const collection: KnowledgeSourceCollection = {
|
|
342
|
-
sources: [],
|
|
343
|
-
skipped: [],
|
|
344
|
-
};
|
|
345
|
-
async function walk(dir: string): Promise<void> {
|
|
346
|
-
for await (const entry of fs.readDir(dir)) {
|
|
347
|
-
const entryPath = join(dir, entry.name);
|
|
348
|
-
if (entry.isDirectory) {
|
|
349
|
-
const skipped = classifyDirectoryOrSkip({ source: entryPath });
|
|
350
|
-
if (skipped != null) {
|
|
351
|
-
collection.skipped.push(skipped);
|
|
352
|
-
continue;
|
|
353
|
-
}
|
|
354
|
-
if (recursive) await walk(entryPath);
|
|
355
|
-
continue;
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
if (!entry.isFile) {
|
|
359
|
-
continue;
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
const skipped = classifySourceOrSkip({ source: entryPath, localSourcePath: entryPath });
|
|
363
|
-
if (skipped != null) {
|
|
364
|
-
collection.skipped.push(skipped);
|
|
365
|
-
continue;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
collection.sources.push({ kind: "local", input: root, localPath: entryPath });
|
|
369
|
-
}
|
|
370
|
-
}
|
|
371
|
-
|
|
372
|
-
await walk(root);
|
|
373
|
-
collection.sources.sort((left, right) => left.localPath.localeCompare(right.localPath));
|
|
374
|
-
collection.skipped.sort((left, right) => left.source.localeCompare(right.source));
|
|
375
|
-
return collection;
|
|
229
|
+
return commandHelpers.collectLocalFiles(root, recursive);
|
|
376
230
|
}
|
|
377
231
|
|
|
378
232
|
function buildSourceReference(source: KnowledgeSource): string {
|
|
379
|
-
return source
|
|
380
|
-
? formatKnowledgeUploadSource(source.uploadPath)
|
|
381
|
-
: source.localPath;
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
export function buildSuggestedSlug(source: KnowledgeSource, index: number): string {
|
|
385
|
-
const normalized = normalize(
|
|
386
|
-
source.kind === "upload" ? source.uploadPath : source.localPath,
|
|
387
|
-
).replace(/\\/g, "/");
|
|
388
|
-
|
|
389
|
-
let stripped: string;
|
|
390
|
-
if (source.kind === "upload") {
|
|
391
|
-
stripped = normalized
|
|
392
|
-
.replace(/^\/workspace\/uploads\//, "")
|
|
393
|
-
.replace(/^\/workspace\//, "")
|
|
394
|
-
.replace(/^uploads\//, "")
|
|
395
|
-
.replace(/\.[^.]+$/, "");
|
|
396
|
-
} else if (normalized.startsWith("/workspace/uploads/")) {
|
|
397
|
-
stripped = normalized.replace(/^\/workspace\/uploads\//, "").replace(/\.[^.]+$/, "");
|
|
398
|
-
} else if (normalized.startsWith("/workspace/")) {
|
|
399
|
-
stripped = normalized.replace(/^\/workspace\//, "").replace(/\.[^.]+$/, "");
|
|
400
|
-
} else if (normalized.startsWith("/")) {
|
|
401
|
-
stripped = basename(normalized, extname(normalized));
|
|
402
|
-
} else {
|
|
403
|
-
stripped = normalized.replace(/\.[^.]+$/, "");
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
// Strip Studio's chat-upload prefix from the filename portion so that
|
|
407
|
-
// knowledge files use the original clean filename.
|
|
408
|
-
const lastSlash = stripped.lastIndexOf("/");
|
|
409
|
-
if (lastSlash >= 0) {
|
|
410
|
-
const dir = stripped.slice(0, lastSlash + 1);
|
|
411
|
-
const file = stripChatUploadPrefix(stripped.slice(lastSlash + 1));
|
|
412
|
-
stripped = file ? `${dir}${file}` : stripped;
|
|
413
|
-
} else {
|
|
414
|
-
stripped = stripChatUploadPrefix(stripped) || stripped;
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
return slugify(stripped || basename(normalized, extname(normalized)) || `document-${index + 1}`);
|
|
418
|
-
}
|
|
419
|
-
export function ensureUniqueSlugs(sources: KnowledgeSource[]): string[] {
|
|
420
|
-
const counts = new Map<string, number>();
|
|
421
|
-
return sources.map((source, index) => {
|
|
422
|
-
const baseSlug = buildSuggestedSlug(source, index);
|
|
423
|
-
const nextCount = (counts.get(baseSlug) ?? 0) + 1;
|
|
424
|
-
counts.set(baseSlug, nextCount);
|
|
425
|
-
return nextCount === 1 ? baseSlug : `${baseSlug}-${nextCount}`;
|
|
426
|
-
});
|
|
427
|
-
}
|
|
428
|
-
|
|
429
|
-
export function deriveKnowledgeRemotePath(
|
|
430
|
-
outputPath: string,
|
|
431
|
-
outputDir: string,
|
|
432
|
-
knowledgePath: string,
|
|
433
|
-
): string {
|
|
434
|
-
const relativeOutputPath = relative(outputDir, outputPath).replace(/\\/g, "/");
|
|
435
|
-
if (!relativeOutputPath || relativeOutputPath.startsWith("..")) {
|
|
436
|
-
throw new Error(`Output path is outside output directory: ${outputPath}`);
|
|
437
|
-
}
|
|
438
|
-
const prefix = normalizeKnowledgeInputPath(knowledgePath);
|
|
439
|
-
const normalizedRelative = normalize(relativeOutputPath).replace(/^\/+/, "");
|
|
440
|
-
return `${prefix}/${normalizedRelative}`.replace(/\\/g, "/");
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
export function createKnowledgeIngestResult(input: {
|
|
444
|
-
source: string;
|
|
445
|
-
localSourcePath: string;
|
|
446
|
-
outputPath: string;
|
|
447
|
-
remotePath: string;
|
|
448
|
-
parser: Pick<KnowledgeParserResult, "slug" | "stats" | "warnings" | "source_type" | "summary">;
|
|
449
|
-
}): KnowledgeIngestFileResult {
|
|
450
|
-
return {
|
|
451
|
-
source: input.source,
|
|
452
|
-
localSourcePath: input.localSourcePath,
|
|
453
|
-
outputPath: input.outputPath,
|
|
454
|
-
remotePath: input.remotePath,
|
|
455
|
-
slug: input.parser.slug,
|
|
456
|
-
sourceType: input.parser.source_type,
|
|
457
|
-
summary: input.parser.summary,
|
|
458
|
-
stats: input.parser.stats,
|
|
459
|
-
warnings: input.parser.warnings,
|
|
460
|
-
};
|
|
233
|
+
return commandHelpers.buildSourceReference(source);
|
|
461
234
|
}
|
|
462
235
|
|
|
463
236
|
export async function runKnowledgeParser(input: {
|
|
@@ -45,6 +45,7 @@ import { skillsHelp } from "../commands/skills/command-help.js";
|
|
|
45
45
|
import { configHelp } from "../commands/config/command-help.js";
|
|
46
46
|
import { openHelp } from "../commands/open/command-help.js";
|
|
47
47
|
import { completionsHelp } from "../commands/completions/command-help.js";
|
|
48
|
+
import { extensionHelp } from "../commands/extension/command-help.js";
|
|
48
49
|
|
|
49
50
|
/**
|
|
50
51
|
* Central registry of all command help definitions.
|
|
@@ -89,4 +90,5 @@ export const COMMANDS: CommandRegistry = {
|
|
|
89
90
|
config: configHelp,
|
|
90
91
|
open: openHelp,
|
|
91
92
|
completions: completionsHelp,
|
|
93
|
+
extension: extensionHelp,
|
|
92
94
|
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
function encodeFilePath(path: string): string {
|
|
2
|
+
return path.split("/").map(encodeURIComponent).join("/");
|
|
3
|
+
}
|
|
4
|
+
|
|
5
|
+
function getBranchPath(branch?: string): string {
|
|
6
|
+
return branch ? `/branches/${branch}` : "";
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function getBranchParam(branch?: string): string {
|
|
10
|
+
return branch ? `?branch_id=${branch}` : "";
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function buildProjectApiPath(project: string, resource: string, branch?: string): string {
|
|
14
|
+
const normalizedResource = resource.startsWith("/") ? resource.slice(1) : resource;
|
|
15
|
+
return `/${project}${getBranchPath(branch)}/${normalizedResource}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function buildProjectFilePath(project: string, filePath: string, branch?: string): string {
|
|
19
|
+
return buildProjectApiPath(project, `files/${encodeFilePath(filePath)}`, branch);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function slugToName(slug: string): string {
|
|
23
|
+
return slug
|
|
24
|
+
.split("-")
|
|
25
|
+
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
|
26
|
+
.join(" ");
|
|
27
|
+
}
|