veryfront 0.1.216 → 0.1.217

package/src/cli/commands/generate/integration-generator.ts

@@ -12,6 +12,18 @@ import { createFileSystem, type FileSystem } from "../../../src/platform/index.j
 import { ensureDir } from "../../utils/fs.js";
 import { isInteractive as checkIsInteractive, promptSync } from "../../../src/platform/index.js";
 import { isCiEnv, isDenoTestingEnv } from "../../../src/config/index.js";
+import {
+  getNonInteractiveConfig,
+  getToolExecuteBody,
+  getToolInputSchema,
+  type IntegrationConfig,
+  normalizeTokenAuthMethod,
+  parseAdditionalAuthParams,
+  parseBooleanOption,
+  parseScopes,
+  type TokenAuthMethod,
+  validateIntegrationName,
+} from "./integration-generator-helpers.js";
 import { select } from "../../utils/terminal-select.js";
 
 let fs: FileSystem;
@@ -41,27 +53,6 @@ export interface IntegrationGeneratorOptions {
   skipPrompts?: boolean;
 }
 
-type TokenAuthMethod =
-  | "basic"
-  | "body"
-  | "client_secret_basic"
-  | "client_secret_post"
-  | "request_body";
-
-interface IntegrationConfig {
-  name: string;
-  displayName: string;
-  authType: "oauth2" | "api-key";
-  apiBaseUrl: string;
-  authorizationUrl?: string;
-  tokenUrl?: string;
-  scopes: string[];
-  tokenAuthMethod: TokenAuthMethod;
-  additionalAuthParams: Record<string, string>;
-  usePKCE: boolean;
-  envVarPrefix: string;
-}
-
 function canRunPrompts(): boolean {
   return !(isCiEnv() || isDenoTestingEnv()) && checkIsInteractive();
 }
@@ -73,86 +64,6 @@ function promptText(question: string, defaultValue?: string): Promise<string> {
   return Promise.resolve(input?.trim() || defaultValue || "");
 }
 
-function parseScopes(scopes?: string): string[] {
-  return scopes?.split(",").map((s) => s.trim()) || [];
-}
-
-function parseAdditionalAuthParams(params?: string): Record<string, string> {
-  if (!params?.trim()) return {};
-
-  return Object.fromEntries(
-    params.split(",")
-      .map((entry) => entry.trim())
-      .filter(Boolean)
-      .map((entry) => {
-        const separatorIndex = entry.indexOf("=");
-        if (separatorIndex === -1) {
-          throw new Error(
-            "Additional auth params must use key=value pairs separated by commas",
-          );
-        }
-
-        const key = entry.slice(0, separatorIndex).trim();
-        const value = entry.slice(separatorIndex + 1).trim();
-
-        if (!key || !value) {
-          throw new Error(
-            "Additional auth params must use non-empty key=value pairs",
-          );
-        }
-
-        return [key, value];
-      }),
-  );
-}
-
-function normalizeTokenAuthMethod(method?: string): TokenAuthMethod {
-  switch (method?.trim().toLowerCase()) {
-    case "basic":
-      return "basic";
-    case "body":
-      return "body";
-    case "client_secret_basic":
-      return "client_secret_basic";
-    case "client_secret_post":
-      return "client_secret_post";
-    case "request_body":
-    case undefined:
-    case "":
-      return "request_body";
-    default:
-      throw new Error(
-        "OAuth token auth method must be one of: request_body, body, basic, client_secret_basic, client_secret_post",
-      );
-  }
-}
-
-function parseBooleanOption(value: string | boolean | undefined, defaultValue = false): boolean {
-  if (typeof value === "boolean") return value;
-  if (!value?.trim()) return defaultValue;
-
-  switch (value.trim().toLowerCase()) {
-    case "y":
-    case "yes":
-    case "true":
-    case "1":
-      return true;
-    case "n":
-    case "no":
-    case "false":
-    case "0":
-      return false;
-    default:
-      throw new Error("Boolean option must be yes/no, true/false, or 1/0");
-  }
-}
-
-function validateIntegrationName(name: string): void {
-  if (!name || !/^[a-z][a-z0-9-]*$/.test(name)) {
-    throw new Error("Integration name must be lowercase letters, numbers, and hyphens");
-  }
-}
-
 export async function generateIntegration(
   projectDir: string,
   options: IntegrationGeneratorOptions = {},
@@ -182,32 +93,6 @@ export async function generateIntegration(
   console.log("");
 }
 
-function getNonInteractiveConfig(options: IntegrationGeneratorOptions): IntegrationConfig {
-  const { name, displayName, authType } = options;
-
-  if (!name || !displayName || !authType) {
-    throw new Error(
-      "Non-interactive mode requires --name, --display-name, and --auth-type options",
-    );
-  }
-
-  const normalizedName = name.toLowerCase();
-
-  return {
-    name: normalizedName,
-    displayName,
-    authType,
-    apiBaseUrl: options.apiBaseUrl ?? `https://api.${name}.com`,
-    authorizationUrl: options.authorizationUrl,
-    tokenUrl: options.tokenUrl,
-    scopes: parseScopes(options.scopes),
-    tokenAuthMethod: normalizeTokenAuthMethod(options.tokenAuthMethod),
-    additionalAuthParams: parseAdditionalAuthParams(options.additionalAuthParams),
-    usePKCE: parseBooleanOption(options.usePKCE, false),
-    envVarPrefix: name.toUpperCase().replace(/-/g, "_"),
-  };
-}
-
 async function getInteractiveConfig(
   options: IntegrationGeneratorOptions,
 ): Promise<IntegrationConfig> {
@@ -809,47 +694,6 @@ export async function searchItems(query: string): Promise<unknown[]> {
   cliLogger.debug("Created API client");
 }
 
-function getToolInputSchema(toolFile: string): string {
-  return TOOL_FILE_CONTENTS[toolFile]?.inputSchema ?? "";
-}
-
-function getToolExecuteBody(toolFile: string): string {
-  return TOOL_FILE_CONTENTS[toolFile]?.executeBody ?? "";
-}
-
-const TOOL_FILE_CONTENTS: Record<string, { inputSchema: string; executeBody: string }> = {
-  "list-items.ts": {
-    inputSchema: `limit: z.number().optional().describe("Maximum number of items to return"),
-    offset: z.number().optional().describe("Number of items to skip"),`,
-    executeBody: `const items = await listItems({
-        limit: input.limit,
-        offset: input.offset,
-      });
-      return {
-        success: true,
-        items,
-        count: items.length,
-      };`,
-  },
-  "get-item.ts": {
-    inputSchema: `id: z.string().describe("The ID of the item to retrieve"),`,
-    executeBody: `const item = await getItem(input.id);
-      return {
-        success: true,
-        item,
-      };`,
-  },
-  "search.ts": {
-    inputSchema: `query: z.string().describe("Search query"),`,
-    executeBody: `const results = await searchItems(input.query);
-      return {
-        success: true,
-        results,
-        count: results.length,
-      };`,
-  },
-};
-
 async function createToolSkeletons(baseDir: string, config: IntegrationConfig): Promise<void> {
   const tools = [
     {

package/src/cli/commands/knowledge/command-helpers.ts

@@ -0,0 +1,295 @@
+import { createFileSystem } from "../../../src/platform/index.js";
+import { basename, extname, join, normalize, relative } from "../../../src/platform/compat/path/index.js";
+import { classifyKnowledgeDirectoryPath, classifyKnowledgeSourcePath } from "./source-policy.js";
+import type {
+  KnowledgeIngestFailedFileResult,
+  KnowledgeIngestFileResult,
+  KnowledgeIngestSkippedFileResult,
+} from "./result.js";
+
+export type KnowledgeSource =
+  | { kind: "local"; input: string; localPath: string }
+  | { kind: "upload"; input: string; uploadPath: string; localPath: string };
+
+export interface KnowledgeSourceCollection {
+  sources: KnowledgeSource[];
+  skipped: KnowledgeIngestSkippedFileResult[];
+}
+
+function slugify(value: string): string {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "document";
+}
+
+const CHAT_UPLOAD_PREFIX_RE =
+  /^chat-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-\d+-[a-z0-9]+-/i;
+
+export function normalizeKnowledgeInputPath(inputPath: string): string {
+  const normalizedPath = normalize(inputPath).replace(/^\/+/, "").replace(/\\/g, "/");
+  if (!normalizedPath || normalizedPath.startsWith("..") || normalizedPath.startsWith("/")) {
+    throw new Error(`Invalid knowledge input path: ${inputPath}`);
+  }
+  return normalizedPath;
+}
+
+export function normalizeProjectUploadPath(inputPath: string): string {
+  return normalizeKnowledgeInputPath(inputPath);
+}
+
+export function formatKnowledgeUploadSource(uploadPath: string): string {
+  const normalizedPath = normalizeKnowledgeInputPath(uploadPath);
+  return normalizedPath === "uploads" || normalizedPath.startsWith("uploads/")
+    ? normalizedPath
+    : `uploads/${normalizedPath}`;
+}
+
+export function resolveExplicitUploadPath(inputPath: string): string {
+  const normalizedInput = normalizeKnowledgeInputPath(inputPath);
+  const displayInput = inputPath.replace(/\\/g, "/");
+  const uploadPath = normalizeProjectUploadPath(inputPath);
+  if (
+    !uploadPath || uploadPath === "uploads" || normalizedInput === "uploads" ||
+    normalizedInput.endsWith("/")
+  ) {
+    throw new Error(
+      `Directory upload references require --path <prefix> --all: ${displayInput}`,
+    );
+  }
+  return uploadPath;
+}
+
+export function isLikelyLocalPath(value: string): boolean {
+  return value.startsWith("/") || value.startsWith("./") || value.startsWith("../") ||
+    /^[A-Za-z]:[\\/]/.test(value);
+}
+
+export function isProjectUploadReference(value: string): boolean {
+  if (isLikelyLocalPath(value)) return false;
+  const normalizedValue = normalize(value).replace(/\\/g, "/").replace(/^\/+/, "");
+  return normalizedValue === "uploads" || normalizedValue.startsWith("uploads/");
+}
+
+export function stripChatUploadPrefix(name: string): string {
+  return name.replace(CHAT_UPLOAD_PREFIX_RE, "");
+}
+
+export function resolveKnowledgeDownloadOutputDir(outputDir: string): string {
+  return join(outputDir, ".uploads");
+}
+
+function createSkippedKnowledgeSource(input: {
+  source: string;
+  localSourcePath?: string | null;
+  message: string;
+  reason: KnowledgeIngestSkippedFileResult["reason"];
+}): KnowledgeIngestSkippedFileResult {
+  return {
+    source: input.source,
+    localSourcePath: input.localSourcePath ?? null,
+    message: input.message,
+    reason: input.reason,
+  };
+}
+
+export function createFailedKnowledgeSource(input: {
+  source: string;
+  localSourcePath: string;
+  message: string;
+  reason: KnowledgeIngestFailedFileResult["reason"];
+}): KnowledgeIngestFailedFileResult {
+  return {
+    source: input.source,
+    localSourcePath: input.localSourcePath,
+    message: input.message,
+    reason: input.reason,
+  };
+}
+
+function classifySourceOrSkip(input: {
+  source: string;
+  localSourcePath?: string | null;
+}): KnowledgeIngestSkippedFileResult | null {
+  const decision = classifyKnowledgeSourcePath(input.source);
+  if (decision.kind === "ingest") {
+    return null;
+  }
+
+  return createSkippedKnowledgeSource({
+    source: input.source,
+    localSourcePath: input.localSourcePath,
+    message: decision.message,
+    reason: decision.reason,
+  });
+}
+
+function classifyDirectoryOrSkip(
+  input: { source: string },
+): KnowledgeIngestSkippedFileResult | null {
+  const decision = classifyKnowledgeDirectoryPath(input.source);
+  if (decision.kind === "ingest") {
+    return null;
+  }
+
+  return createSkippedKnowledgeSource({
+    source: input.source,
+    localSourcePath: null,
+    message: decision.message,
+    reason: decision.reason,
+  });
+}
+
+export async function collectLocalFiles(
+  root: string,
+  recursive: boolean,
+): Promise<KnowledgeSourceCollection> {
+  const fs = createFileSystem();
+  const stat = await fs.stat(root);
+  if (stat.isFile) {
+    const skipped = classifySourceOrSkip({ source: root, localSourcePath: root });
+    return skipped == null
+      ? {
+        sources: [{ kind: "local", input: root, localPath: root }],
+        skipped: [],
+      }
+      : {
+        sources: [],
+        skipped: [skipped],
+      };
+  }
+  if (!stat.isDirectory) {
+    return { sources: [], skipped: [] };
+  }
+
+  const skippedRootDirectory = classifyDirectoryOrSkip({ source: root });
+  if (skippedRootDirectory != null) {
+    return {
+      sources: [],
+      skipped: [skippedRootDirectory],
+    };
+  }
+
+  const collection: KnowledgeSourceCollection = {
+    sources: [],
+    skipped: [],
+  };
+  async function walk(dir: string): Promise<void> {
+    for await (const entry of fs.readDir(dir)) {
+      const entryPath = join(dir, entry.name);
+      if (entry.isDirectory) {
+        const skipped = classifyDirectoryOrSkip({ source: entryPath });
+        if (skipped != null) {
+          collection.skipped.push(skipped);
+          continue;
+        }
+        if (recursive) await walk(entryPath);
+        continue;
+      }
+
+      if (!entry.isFile) {
+        continue;
+      }
+
+      const skipped = classifySourceOrSkip({ source: entryPath, localSourcePath: entryPath });
+      if (skipped != null) {
+        collection.skipped.push(skipped);
+        continue;
+      }
+
+      collection.sources.push({ kind: "local", input: root, localPath: entryPath });
+    }
+  }
+
+  await walk(root);
+  collection.sources.sort((left, right) => left.localPath.localeCompare(right.localPath));
+  collection.skipped.sort((left, right) => left.source.localeCompare(right.source));
+  return collection;
+}
+
+export function buildSourceReference(source: KnowledgeSource): string {
+  return source.kind === "upload"
+    ? formatKnowledgeUploadSource(source.uploadPath)
+    : source.localPath;
+}
+
+export function buildSuggestedSlug(source: KnowledgeSource, index: number): string {
+  const normalized = normalize(
+    source.kind === "upload" ? source.uploadPath : source.localPath,
+  ).replace(/\\/g, "/");
+
+  let stripped: string;
+  if (source.kind === "upload") {
+    stripped = normalized
+      .replace(/^\/workspace\/uploads\//, "")
+      .replace(/^\/workspace\//, "")
+      .replace(/^uploads\//, "")
+      .replace(/\.[^.]+$/, "");
+  } else if (normalized.startsWith("/workspace/uploads/")) {
+    stripped = normalized.replace(/^\/workspace\/uploads\//, "").replace(/\.[^.]+$/, "");
+  } else if (normalized.startsWith("/workspace/")) {
+    stripped = normalized.replace(/^\/workspace\//, "").replace(/\.[^.]+$/, "");
+  } else if (normalized.startsWith("/")) {
+    stripped = basename(normalized, extname(normalized));
+  } else {
+    stripped = normalized.replace(/\.[^.]+$/, "");
+  }
+
+  const lastSlash = stripped.lastIndexOf("/");
+  if (lastSlash >= 0) {
+    const dir = stripped.slice(0, lastSlash + 1);
+    const file = stripChatUploadPrefix(stripped.slice(lastSlash + 1));
+    stripped = file ? `${dir}${file}` : stripped;
+  } else {
+    stripped = stripChatUploadPrefix(stripped) || stripped;
+  }
+
+  return slugify(stripped || basename(normalized, extname(normalized)) || `document-${index + 1}`);
+}
+
+export function ensureUniqueSlugs(sources: KnowledgeSource[]): string[] {
+  const counts = new Map<string, number>();
+  return sources.map((source, index) => {
+    const baseSlug = buildSuggestedSlug(source, index);
+    const nextCount = (counts.get(baseSlug) ?? 0) + 1;
+    counts.set(baseSlug, nextCount);
+    return nextCount === 1 ? baseSlug : `${baseSlug}-${nextCount}`;
+  });
+}
+
+export function deriveKnowledgeRemotePath(
+  outputPath: string,
+  outputDir: string,
+  knowledgePath: string,
+): string {
+  const relativeOutputPath = relative(outputDir, outputPath).replace(/\\/g, "/");
+  if (!relativeOutputPath || relativeOutputPath.startsWith("..")) {
+    throw new Error(`Output path is outside output directory: ${outputPath}`);
+  }
+  const prefix = normalizeKnowledgeInputPath(knowledgePath);
+  const normalizedRelative = normalize(relativeOutputPath).replace(/^\/+/, "");
+  return `${prefix}/${normalizedRelative}`.replace(/\\/g, "/");
+}
+
+export function createKnowledgeIngestResult(input: {
+  source: string;
+  localSourcePath: string;
+  outputPath: string;
+  remotePath: string;
+  parser: {
+    slug: string;
+    stats: Record<string, unknown>;
+    warnings: string[];
+    source_type: string;
+    summary: string;
+  };
+}): KnowledgeIngestFileResult {
+  return {
+    source: input.source,
+    localSourcePath: input.localSourcePath,
+    outputPath: input.outputPath,
+    remotePath: input.remotePath,
+    slug: input.parser.slug,
+    sourceType: input.parser.source_type,
+    summary: input.parser.summary,
+    stats: input.parser.stats,
+    warnings: input.parser.warnings,
+  };
+}