veryfront 0.1.216 → 0.1.217

package/src/cli/commands/knowledge/command.ts

@@ -2,7 +2,7 @@ import * as dntShim from "../../../_dnt.shims.js";
 import { z } from "zod";
 type SafeParseResult<T> = { success: true; data: T } | { success: false; error: z.ZodError };
 import { type CommandResult, createFileSystem, getEnv, runCommand } from "../../../src/platform/index.js";
-import { basename, extname, join, normalize, relative } from "../../../src/platform/compat/path/index.js";
+import { basename } from "../../../src/platform/compat/path/index.js";
 import { withSpan } from "../../../src/observability/tracing/otlp-setup.js";
 import { cliLogger } from "../../utils/index.js";
 import { type ApiClient, createApiClient, resolveConfigWithAuth } from "../../shared/config.js";
@@ -11,9 +11,10 @@ import { getStringArg } from "../../shared/parsed-args.js";
 import { downloadUploadToFile, listAllUploads, type UploadItem } from "../uploads/command.js";
 import { putRemoteFileFromLocal } from "../files/command.js";
 import { knowledgeIngestPythonSource } from "./parser-source.js";
+import * as commandHelpers from "./command-helpers.js";
 import { createJobUserLogger, type Logger, serverLogger } from "../../../src/utils/index.js";
 import { writeJobResultIfConfigured } from "../../utils/write-job-result.js";
-import { classifyKnowledgeDirectoryPath, classifyKnowledgeSourcePath } from "./source-policy.js";
+import { classifyKnowledgeSourcePath } from "./source-policy.js";
 import {
   buildKnowledgeIngestJobResult,
   type KnowledgeIngestFailedFileResult,
@@ -172,108 +173,10 @@ export function parseKnowledgeIngestArgs(
   });
 }
 
-export function normalizeKnowledgeInputPath(inputPath: string): string {
-  const normalizedPath = normalize(inputPath).replace(/^\/+/, "").replace(/\\/g, "/");
-  if (!normalizedPath || normalizedPath.startsWith("..") || normalizedPath.startsWith("/")) {
-    throw new Error(`Invalid knowledge input path: ${inputPath}`);
-  }
-  return normalizedPath;
-}
-
-export function normalizeProjectUploadPath(inputPath: string): string {
-  return normalizeKnowledgeInputPath(inputPath);
-}
-
-export function formatKnowledgeUploadSource(uploadPath: string): string {
-  const normalizedPath = normalizeKnowledgeInputPath(uploadPath);
-  return normalizedPath === "uploads" || normalizedPath.startsWith("uploads/")
-    ? normalizedPath
-    : `uploads/${normalizedPath}`;
-}
-
-function resolveExplicitUploadPath(inputPath: string): string {
-  const normalizedInput = normalizeKnowledgeInputPath(inputPath);
-  const displayInput = inputPath.replace(/\\/g, "/");
-  const uploadPath = normalizeProjectUploadPath(inputPath);
-  if (
-    !uploadPath || uploadPath === "uploads" || normalizedInput === "uploads" ||
-    normalizedInput.endsWith("/")
-  ) {
-    throw new Error(
-      `Directory upload references require --path <prefix> --all: ${displayInput}`,
-    );
-  }
-  return uploadPath;
-}
-
-export function isLikelyLocalPath(value: string): boolean {
-  return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") ||
-    /^[A-Za-z]:[\\/]/.test(value);
-}
-
-function isProjectUploadReference(value: string): boolean {
-  if (isLikelyLocalPath(value)) return false;
-  const normalizedValue = normalize(value).replace(/\\/g, "/").replace(/^\/+/, "");
-  return normalizedValue === "uploads" || normalizedValue.startsWith("uploads/");
-}
-
-function slugify(value: string): string {
-  return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "document";
-}
-
-/**
- * Strip the chat-upload prefix that Studio prepends to uploaded files.
- *
- * Studio stores uploads with a generated prefix like:
- *   chat-<uuid>-<timestamp>-<shortid>-<original-filename>
- *
- * This function removes the prefix so knowledge files use the clean
- * original filename (e.g. "agents" instead of
- * "chat-909d3dbc-5a9a-4156-97e4-bcceb5c2ec0d-1773942180291-fv1qg5-agents").
- */
-const CHAT_UPLOAD_PREFIX_RE =
-  /^chat-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-\d+-[a-z0-9]+-/i;
-
-export function stripChatUploadPrefix(name: string): string {
-  return name.replace(CHAT_UPLOAD_PREFIX_RE, "");
-}
-
 function defaultOutputRoot(): Promise<string> {
   return dntShim.Deno.makeTempDir({ prefix: "veryfront-knowledge-" });
 }
 
-export function resolveKnowledgeDownloadOutputDir(outputDir: string): string {
-  return join(outputDir, ".uploads");
-}
-
-function createSkippedKnowledgeSource(input: {
-  source: string;
-  localSourcePath?: string | null;
-  message: string;
-  reason: KnowledgeIngestSkippedFileResult["reason"];
-}): KnowledgeIngestSkippedFileResult {
-  return {
-    source: input.source,
-    localSourcePath: input.localSourcePath ?? null,
-    message: input.message,
-    reason: input.reason,
-  };
-}
-
-function createFailedKnowledgeSource(input: {
-  source: string;
-  localSourcePath: string;
-  message: string;
-  reason: KnowledgeIngestFailedFileResult["reason"];
-}): KnowledgeIngestFailedFileResult {
-  return {
-    source: input.source,
-    localSourcePath: input.localSourcePath,
-    message: input.message,
-    reason: input.reason,
-  };
-}
-
 function classifySourceOrSkip(input: {
   source: string;
   localSourcePath?: string | null;
@@ -283,180 +186,51 @@ function classifySourceOrSkip(input: {
     return null;
   }
 
-  return createSkippedKnowledgeSource({
+  return {
     source: input.source,
-    localSourcePath: input.localSourcePath,
+    localSourcePath: input.localSourcePath ?? null,
     message: decision.message,
     reason: decision.reason,
-  });
+  };
 }
 
-function classifyDirectoryOrSkip(input: {
-  source: string;
-}): KnowledgeIngestSkippedFileResult | null {
-  const decision = classifyKnowledgeDirectoryPath(input.source);
-  if (decision.kind === "ingest") {
-    return null;
-  }
+export const normalizeKnowledgeInputPath = commandHelpers.normalizeKnowledgeInputPath;
+export const normalizeProjectUploadPath = commandHelpers.normalizeProjectUploadPath;
+export const formatKnowledgeUploadSource = commandHelpers.formatKnowledgeUploadSource;
+export const isLikelyLocalPath = commandHelpers.isLikelyLocalPath;
+export const stripChatUploadPrefix = commandHelpers.stripChatUploadPrefix;
+export const resolveKnowledgeDownloadOutputDir = commandHelpers.resolveKnowledgeDownloadOutputDir;
+export const buildSuggestedSlug = commandHelpers.buildSuggestedSlug;
+export const ensureUniqueSlugs = commandHelpers.ensureUniqueSlugs;
+export const deriveKnowledgeRemotePath = commandHelpers.deriveKnowledgeRemotePath;
+export const createKnowledgeIngestResult = commandHelpers.createKnowledgeIngestResult;
 
-  return createSkippedKnowledgeSource({
-    source: input.source,
-    localSourcePath: null,
-    message: decision.message,
-    reason: decision.reason,
-  });
+function resolveExplicitUploadPath(inputPath: string): string {
+  return commandHelpers.resolveExplicitUploadPath(inputPath);
+}
+
+function isProjectUploadReference(value: string): boolean {
+  return commandHelpers.isProjectUploadReference(value);
+}
+
+function createFailedKnowledgeSource(input: {
+  source: string;
+  localSourcePath: string;
+  message: string;
+  reason: KnowledgeIngestFailedFileResult["reason"];
+}): KnowledgeIngestFailedFileResult {
+  return commandHelpers.createFailedKnowledgeSource(input);
 }
 
 async function collectLocalFiles(
   root: string,
   recursive: boolean,
 ): Promise<KnowledgeSourceCollection> {
-  const fs = createFileSystem();
-  const stat = await fs.stat(root);
-  if (stat.isFile) {
-    const skipped = classifySourceOrSkip({ source: root, localSourcePath: root });
-    return skipped == null
-      ? {
-        sources: [{ kind: "local", input: root, localPath: root }],
-        skipped: [],
-      }
-      : {
-        sources: [],
-        skipped: [skipped],
-      };
-  }
-  if (!stat.isDirectory) {
-    return { sources: [], skipped: [] };
-  }
-
-  const skippedRootDirectory = classifyDirectoryOrSkip({ source: root });
-  if (skippedRootDirectory != null) {
-    return {
-      sources: [],
-      skipped: [skippedRootDirectory],
-    };
-  }
-
-  const collection: KnowledgeSourceCollection = {
-    sources: [],
-    skipped: [],
-  };
-  async function walk(dir: string): Promise<void> {
-    for await (const entry of fs.readDir(dir)) {
-      const entryPath = join(dir, entry.name);
-      if (entry.isDirectory) {
-        const skipped = classifyDirectoryOrSkip({ source: entryPath });
-        if (skipped != null) {
-          collection.skipped.push(skipped);
-          continue;
-        }
-        if (recursive) await walk(entryPath);
-        continue;
-      }
-
-      if (!entry.isFile) {
-        continue;
-      }
-
-      const skipped = classifySourceOrSkip({ source: entryPath, localSourcePath: entryPath });
-      if (skipped != null) {
-        collection.skipped.push(skipped);
-        continue;
-      }
-
-      collection.sources.push({ kind: "local", input: root, localPath: entryPath });
-    }
-  }
-
-  await walk(root);
-  collection.sources.sort((left, right) => left.localPath.localeCompare(right.localPath));
-  collection.skipped.sort((left, right) => left.source.localeCompare(right.source));
-  return collection;
+  return commandHelpers.collectLocalFiles(root, recursive);
 }
 
 function buildSourceReference(source: KnowledgeSource): string {
-  return source.kind === "upload"
-    ? formatKnowledgeUploadSource(source.uploadPath)
-    : source.localPath;
-}
-
-export function buildSuggestedSlug(source: KnowledgeSource, index: number): string {
-  const normalized = normalize(
-    source.kind === "upload" ? source.uploadPath : source.localPath,
-  ).replace(/\\/g, "/");
-
-  let stripped: string;
-  if (source.kind === "upload") {
-    stripped = normalized
-      .replace(/^\/workspace\/uploads\//, "")
-      .replace(/^\/workspace\//, "")
-      .replace(/^uploads\//, "")
-      .replace(/\.[^.]+$/, "");
-  } else if (normalized.startsWith("/workspace/uploads/")) {
-    stripped = normalized.replace(/^\/workspace\/uploads\//, "").replace(/\.[^.]+$/, "");
-  } else if (normalized.startsWith("/workspace/")) {
-    stripped = normalized.replace(/^\/workspace\//, "").replace(/\.[^.]+$/, "");
-  } else if (normalized.startsWith("/")) {
-    stripped = basename(normalized, extname(normalized));
-  } else {
-    stripped = normalized.replace(/\.[^.]+$/, "");
-  }
-
-  // Strip Studio's chat-upload prefix from the filename portion so that
-  // knowledge files use the original clean filename.
-  const lastSlash = stripped.lastIndexOf("/");
-  if (lastSlash >= 0) {
-    const dir = stripped.slice(0, lastSlash + 1);
-    const file = stripChatUploadPrefix(stripped.slice(lastSlash + 1));
-    stripped = file ? `${dir}${file}` : stripped;
-  } else {
-    stripped = stripChatUploadPrefix(stripped) || stripped;
-  }
-
-  return slugify(stripped || basename(normalized, extname(normalized)) || `document-${index + 1}`);
-}
-export function ensureUniqueSlugs(sources: KnowledgeSource[]): string[] {
-  const counts = new Map<string, number>();
-  return sources.map((source, index) => {
-    const baseSlug = buildSuggestedSlug(source, index);
-    const nextCount = (counts.get(baseSlug) ?? 0) + 1;
-    counts.set(baseSlug, nextCount);
-    return nextCount === 1 ? baseSlug : `${baseSlug}-${nextCount}`;
-  });
-}
-
-export function deriveKnowledgeRemotePath(
-  outputPath: string,
-  outputDir: string,
-  knowledgePath: string,
-): string {
-  const relativeOutputPath = relative(outputDir, outputPath).replace(/\\/g, "/");
-  if (!relativeOutputPath || relativeOutputPath.startsWith("..")) {
-    throw new Error(`Output path is outside output directory: ${outputPath}`);
-  }
-  const prefix = normalizeKnowledgeInputPath(knowledgePath);
-  const normalizedRelative = normalize(relativeOutputPath).replace(/^\/+/, "");
-  return `${prefix}/${normalizedRelative}`.replace(/\\/g, "/");
-}
-
-export function createKnowledgeIngestResult(input: {
-  source: string;
-  localSourcePath: string;
-  outputPath: string;
-  remotePath: string;
-  parser: Pick<KnowledgeParserResult, "slug" | "stats" | "warnings" | "source_type" | "summary">;
-}): KnowledgeIngestFileResult {
-  return {
-    source: input.source,
-    localSourcePath: input.localSourcePath,
-    outputPath: input.outputPath,
-    remotePath: input.remotePath,
-    slug: input.parser.slug,
-    sourceType: input.parser.source_type,
-    summary: input.parser.summary,
-    stats: input.parser.stats,
-    warnings: input.parser.warnings,
-  };
+  return commandHelpers.buildSourceReference(source);
 }
 
 export async function runKnowledgeParser(input: {

package/src/cli/mcp/remote-file-tool-helpers.ts

@@ -0,0 +1,27 @@
+function encodeFilePath(path: string): string {
+  return path.split("/").map(encodeURIComponent).join("/");
+}
+
+function getBranchPath(branch?: string): string {
+  return branch ? `/branches/${branch}` : "";
+}
+
+export function getBranchParam(branch?: string): string {
+  return branch ? `?branch_id=${branch}` : "";
+}
+
+export function buildProjectApiPath(project: string, resource: string, branch?: string): string {
+  const normalizedResource = resource.startsWith("/") ? resource.slice(1) : resource;
+  return `/${project}${getBranchPath(branch)}/${normalizedResource}`;
+}
+
+export function buildProjectFilePath(project: string, filePath: string, branch?: string): string {
+  return buildProjectApiPath(project, `files/${encodeFilePath(filePath)}`, branch);
+}
+
+export function slugToName(slug: string): string {
+  return slug
+    .split("-")
+    .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
+    .join(" ");
+}

package/src/cli/mcp/remote-file-tools.ts

@@ -16,6 +16,12 @@ import { withSpan } from "../../src/observability/tracing/otlp-setup.js";
 import { randomSuffix } from "../shared/slug.js";
 
 import { DEFAULT_LOCAL_API_URL } from "../shared/constants.js";
+import {
+  buildProjectApiPath,
+  buildProjectFilePath,
+  getBranchParam,
+  slugToName,
+} from "./remote-file-tool-helpers.js";
 
 function getApiBaseUrl(): string {
   return getEnvironmentConfig().apiBaseUrl || DEFAULT_LOCAL_API_URL;
@@ -73,27 +79,6 @@ async function apiRequest<T>(
   }
 }
 
-function encodeFilePath(path: string): string {
-  return path.split("/").map(encodeURIComponent).join("/");
-}
-
-function getBranchPath(branch?: string): string {
-  return branch ? `/branches/${branch}` : "";
-}
-
-function getBranchParam(branch?: string): string {
-  return branch ? `?branch_id=${branch}` : "";
-}
-
-function buildProjectApiPath(project: string, resource: string, branch?: string): string {
-  const normalizedResource = resource.startsWith("/") ? resource.slice(1) : resource;
-  return `/${project}${getBranchPath(branch)}/${normalizedResource}`;
-}
-
-function buildProjectFilePath(project: string, filePath: string, branch?: string): string {
-  return buildProjectApiPath(project, `files/${encodeFilePath(filePath)}`, branch);
-}
-
 interface RemoteFile {
   id?: string;
   path: string;
@@ -145,13 +130,6 @@ interface Project {
 
 const MAX_SLUG_ATTEMPTS = 10;
 
-function slugToName(slug: string): string {
-  return slug
-    .split("-")
-    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
-    .join(" ");
-}
-
 /**
  * Create a project with slug conflict retry.
  * On 409, appends a random suffix and retries (matches reserveProjectSlug behavior).

package/src/cli/templates/integration-loader-helpers.ts

@@ -0,0 +1,49 @@
+import { join } from "../../src/fs/index.js";
+import type { IntegrationName, TemplateFile } from "./types.js";
+
+export function resolveIntegrationModuleDir(
+  moduleUrl: string,
+  platform = typeof process !== "undefined" ? process.platform : undefined,
+): string {
+  const normalizedModuleUrl = new URL(".", moduleUrl);
+
+  if (normalizedModuleUrl.protocol !== "file:") return normalizedModuleUrl.href;
+
+  let moduleDir = normalizedModuleUrl.pathname;
+  if (platform === "win32" && moduleDir.startsWith("/")) {
+    moduleDir = moduleDir.slice(1);
+  }
+
+  return moduleDir;
+}
+
+export function buildIntegrationDirectory(
+  moduleDir: string,
+  integrationName: string,
+): string {
+  return join(moduleDir, "integrations", integrationName);
+}
+
+export function buildUnknownIntegrationErrors(
+  integrations: IntegrationName[],
+  availableIntegrations: readonly IntegrationName[],
+): string[] {
+  const availableList = availableIntegrations.join(", ");
+  return integrations
+    .filter((integration) => !availableIntegrations.includes(integration))
+    .map((integration) => `Unknown integration: ${integration}. Available: ${availableList}`);
+}
+
+export function mergeIntegrationFiles(
+  integrations: Array<{ files: TemplateFile[] }>,
+): TemplateFile[] {
+  const fileMap = new Map<string, TemplateFile>();
+
+  for (const integration of integrations) {
+    for (const file of integration.files) {
+      fileMap.set(file.path, file);
+    }
+  }
+
+  return [...fileMap.values()].sort((a, b) => a.path.localeCompare(b.path));
+}

package/src/cli/templates/integration-loader.ts

@@ -10,6 +10,12 @@
 
 import { createFileSystem, join } from "../../src/fs/index.js";
 import { loadTemplateFromDirectory } from "./loader.js";
+import {
+  buildIntegrationDirectory,
+  buildUnknownIntegrationErrors,
+  mergeIntegrationFiles,
+  resolveIntegrationModuleDir,
+} from "./integration-loader-helpers.js";
 import type {
   IntegrationConfig,
   IntegrationName,
@@ -140,23 +146,14 @@ export const USE_CASE_CONFIGS: Record<UseCaseName, UseCaseConfig> = {
 };
 
 function getModuleDir(): string {
-  const moduleUrl = new URL(".", import.meta.url);
-
-  if (moduleUrl.protocol !== "file:") return moduleUrl.href;
-
-  let moduleDir = moduleUrl.pathname;
-  if (typeof process !== "undefined" && process.platform === "win32" && moduleDir.startsWith("/")) {
-    moduleDir = moduleDir.slice(1);
-  }
-
-  return moduleDir;
+  return resolveIntegrationModuleDir(import.meta.url);
 }
 
 /**
  * Get the directory path for an integration
  */
 export function getIntegrationDirectory(integrationName: string): string {
-  return join(getModuleDir(), "integrations", integrationName);
+  return buildIntegrationDirectory(getModuleDir(), integrationName);
 }
 
 /**
@@ -200,15 +197,7 @@ export function validateIntegrations(integrations: IntegrationName[]): {
   valid: boolean;
   errors: string[];
 } {
-  const errors: string[] = [];
-
-  for (const integration of integrations) {
-    if (AVAILABLE_INTEGRATIONS.includes(integration)) continue;
-
-    errors.push(
-      `Unknown integration: ${integration}. Available: ${AVAILABLE_INTEGRATIONS.join(", ")}`,
-    );
-  }
+  const errors = buildUnknownIntegrationErrors(integrations, AVAILABLE_INTEGRATIONS);
 
   return { valid: errors.length === 0, errors };
 }
@@ -225,8 +214,6 @@ export async function loadIntegrations(
 }> {
   const integrations: ResolvedIntegration[] = [];
   const errors: string[] = [];
-  const fileMap = new Map<string, TemplateFile>();
-
   for (const name of integrationNames) {
     const integration = await loadIntegration(name);
     if (!integration) {
@@ -235,16 +222,11 @@ export async function loadIntegrations(
     }
 
     integrations.push(integration);
-
-    for (const file of integration.files) {
-      // Later integrations override earlier ones
-      fileMap.set(file.path, file);
-    }
   }
 
   return {
     integrations,
-    files: [...fileMap.values()].sort((a, b) => a.path.localeCompare(b.path)),
+    files: mergeIntegrationFiles(integrations),
     errors,
   };
 }