veryfront 0.1.83 → 0.1.85

package/src/cli/commands/knowledge/parser-source.ts

@@ -14,6 +14,53 @@ def yaml_quote(value: Any) -> str:
 
 
 CODE_FENCE = chr(96) * 3
+TEXT_FILE_EXTENSIONS = {
+    ".c",
+    ".cc",
+    ".conf",
+    ".cpp",
+    ".cs",
+    ".css",
+    ".go",
+    ".h",
+    ".hpp",
+    ".ini",
+    ".java",
+    ".js",
+    ".jsonl",
+    ".jsx",
+    ".kt",
+    ".less",
+    ".lua",
+    ".mjs",
+    ".ndjson",
+    ".php",
+    ".pl",
+    ".py",
+    ".r",
+    ".rb",
+    ".rs",
+    ".sass",
+    ".scala",
+    ".scss",
+    ".sh",
+    ".sql",
+    ".swift",
+    ".toml",
+    ".ts",
+    ".tsx",
+    ".xml",
+    ".yaml",
+    ".yml",
+    ".zsh",
+}
+TEXT_FILE_NAMES = {
+    "dockerfile",
+    "makefile",
+    "readme",
+    "license",
+    "changelog",
+}
 
 
 def slugify(value: str) -> str:
@@ -406,6 +453,7 @@ def parse_json(path: str):
 
 def select_parser(path: Path):
     ext = path.suffix.lower()
+    name = path.name.lower()
     if ext == ".pdf":
         return "pdf", prefer_kreuzberg("pdf", parse_pdf)
     if ext in {".csv", ".tsv"}:
@@ -424,6 +472,10 @@ def select_parser(path: Path):
         return ext.lstrip("."), parse_text
     if ext == ".json":
         return "json", parse_json
+    if ext in TEXT_FILE_EXTENSIONS:
+        return ext.lstrip("."), parse_text
+    if not ext and name in TEXT_FILE_NAMES:
+        return "text", parse_text
     raise ValueError(f"Unsupported file type: {ext}")
 
 

package/src/cli/commands/knowledge/result.ts

@@ -0,0 +1,88 @@
+export type KnowledgeIngestSkipReason =
+  | "hidden_path"
+  | "ignored_directory"
+  | "unsupported_file_type";
+
+export type KnowledgeIngestFailureReason =
+  | "parser_error"
+  | "upload_error";
+
+export interface KnowledgeIngestFileResult {
+  source: string;
+  localSourcePath: string;
+  outputPath: string;
+  remotePath: string;
+  slug: string;
+  sourceType: string;
+  summary: string;
+  stats: Record<string, unknown>;
+  warnings: string[];
+}
+
+export interface KnowledgeIngestSkippedFileResult {
+  source: string;
+  localSourcePath: string | null;
+  reason: KnowledgeIngestSkipReason;
+  message: string;
+}
+
+export interface KnowledgeIngestFailedFileResult {
+  source: string;
+  localSourcePath: string;
+  reason: KnowledgeIngestFailureReason;
+  message: string;
+}
+
+export interface KnowledgeIngestSummary {
+  requested_count: number;
+  ingested_count: number;
+  skipped_count: number;
+  failed_count: number;
+}
+
+export interface KnowledgeIngestResultMetadata {
+  requested_count: number;
+  source_mode: "explicit_sources" | "path_prefix";
+  knowledge_path: string;
+}
+
+export interface KnowledgeIngestJobResult {
+  kind: "knowledge_ingest";
+  version: 1;
+  metadata: KnowledgeIngestResultMetadata;
+  summary: KnowledgeIngestSummary;
+  ingested: KnowledgeIngestFileResult[];
+  skipped: KnowledgeIngestSkippedFileResult[];
+  failed: KnowledgeIngestFailedFileResult[];
+}
+
+export function buildKnowledgeIngestJobResult(input: {
+  requestedCount: number;
+  sourceMode: KnowledgeIngestResultMetadata["source_mode"];
+  knowledgePath: string;
+  ingested: KnowledgeIngestFileResult[];
+  skipped?: KnowledgeIngestSkippedFileResult[];
+  failed?: KnowledgeIngestFailedFileResult[];
+}): KnowledgeIngestJobResult {
+  const skipped = input.skipped ?? [];
+  const failed = input.failed ?? [];
+
+  return {
+    kind: "knowledge_ingest",
+    version: 1,
+    metadata: {
+      requested_count: input.requestedCount,
+      source_mode: input.sourceMode,
+      knowledge_path: input.knowledgePath,
+    },
+    summary: {
+      requested_count: input.requestedCount,
+      ingested_count: input.ingested.length,
+      skipped_count: skipped.length,
+      failed_count: failed.length,
+    },
+    ingested: input.ingested,
+    skipped,
+    failed,
+  };
+}

package/src/cli/commands/knowledge/source-policy.ts

@@ -0,0 +1,164 @@
+import { basename, extname, normalize } from "../../../src/platform/compat/path/index.js";
+import type { KnowledgeIngestSkipReason } from "./result.js";
+
+const RICH_FILE_EXTENSIONS = new Set([
+  ".pdf",
+  ".csv",
+  ".tsv",
+  ".docx",
+  ".xlsx",
+  ".xls",
+  ".pptx",
+  ".html",
+  ".htm",
+  ".txt",
+  ".json",
+  ".md",
+  ".mdx",
+]);
+
+const TEXT_FALLBACK_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".conf",
+  ".cpp",
+  ".cs",
+  ".css",
+  ".go",
+  ".h",
+  ".hpp",
+  ".ini",
+  ".java",
+  ".js",
+  ".jsonl",
+  ".jsx",
+  ".kt",
+  ".less",
+  ".lua",
+  ".mjs",
+  ".ndjson",
+  ".php",
+  ".pl",
+  ".py",
+  ".r",
+  ".rb",
+  ".rs",
+  ".sass",
+  ".scala",
+  ".scss",
+  ".sh",
+  ".sql",
+  ".swift",
+  ".toml",
+  ".tsx",
+  ".ts",
+  ".xml",
+  ".yaml",
+  ".yml",
+  ".zsh",
+]);
+
+const TEXT_FALLBACK_FILENAMES = new Set([
+  "dockerfile",
+  "makefile",
+  "readme",
+  "license",
+  "changelog",
+]);
+
+const IGNORED_DIRECTORY_NAMES = new Set([
+  "node_modules",
+]);
+
+export type KnowledgeIngestSourceDecision =
+  | { kind: "ingest" }
+  | {
+    kind: "skip";
+    reason: KnowledgeIngestSkipReason;
+    message: string;
+  };
+
+function normalizePathSegments(value: string): string[] {
+  return normalize(value)
+    .replaceAll("\\", "/")
+    .split("/")
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 0 && segment !== "." && segment !== "..");
+}
+
+function findHiddenSegment(segments: string[]): string | null {
+  for (const segment of segments) {
+    if (segment.startsWith(".")) {
+      return segment;
+    }
+  }
+
+  return null;
+}
+
+function findIgnoredDirectory(segments: string[]): string | null {
+  for (const segment of segments) {
+    if (IGNORED_DIRECTORY_NAMES.has(segment.toLowerCase())) {
+      return segment;
+    }
+  }
+
+  return null;
+}
+
+function isSupportedKnowledgeFile(path: string): boolean {
+  const extension = extname(path).toLowerCase();
+  if (
+    extension && (RICH_FILE_EXTENSIONS.has(extension) || TEXT_FALLBACK_EXTENSIONS.has(extension))
+  ) {
+    return true;
+  }
+
+  const fileName = basename(path).toLowerCase();
+  return TEXT_FALLBACK_FILENAMES.has(fileName);
+}
+
+function classifyCommonKnowledgePath(path: string): KnowledgeIngestSourceDecision | null {
+  const segments = normalizePathSegments(path);
+  const hiddenSegment = findHiddenSegment(segments);
+  if (hiddenSegment != null) {
+    return {
+      kind: "skip",
+      reason: "hidden_path",
+      message: `Hidden file or directory skipped: ${hiddenSegment}`,
+    };
+  }
+
+  const ignoredDirectory = findIgnoredDirectory(segments);
+  if (ignoredDirectory != null) {
+    return {
+      kind: "skip",
+      reason: "ignored_directory",
+      message: `Ignored directory skipped: ${ignoredDirectory}`,
+    };
+  }
+
+  return null;
+}
+
+export function classifyKnowledgeDirectoryPath(path: string): KnowledgeIngestSourceDecision {
+  return classifyCommonKnowledgePath(path) ?? { kind: "ingest" };
+}
+
+export function classifyKnowledgeSourcePath(path: string): KnowledgeIngestSourceDecision {
+  const commonDecision = classifyCommonKnowledgePath(path);
+  if (commonDecision != null) {
+    return commonDecision;
+  }
+
+  if (!isSupportedKnowledgeFile(path)) {
+    const extension = extname(path).toLowerCase();
+    return {
+      kind: "skip",
+      reason: "unsupported_file_type",
+      message: `Unsupported file type: ${extension || basename(path)}`,
+    };
+  }
+
+  return { kind: "ingest" };
+}

package/src/deno.js

@@ -1,6 +1,6 @@
 export default {
   "name": "veryfront",
-  "version": "0.1.83",
+  "version": "0.1.85",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "exclude": [

package/src/src/jobs/index.ts

@@ -60,10 +60,16 @@ export {
   JobEventSchema,
   type JobEventsResponse,
   JobEventsResponseSchema,
+  type JobKind,
+  JobKindSchema,
+  type JobListItem,
+  JobListItemSchema,
   type JobLogsResponse,
   JobLogsResponseSchema,
   type JobResult,
   JobResultSchema,
+  type JobResultSummary,
+  JobResultSummarySchema,
   JobSchema,
   type JobStatus,
   JobStatusSchema,
@@ -73,8 +79,18 @@ export {
   JobTargetDefinitionsResponseSchema,
   type KnowledgeIngestBatchSource,
   KnowledgeIngestBatchSourceSchema,
+  type KnowledgeIngestBatchSourceWithMessage,
+  KnowledgeIngestBatchSourceWithMessageSchema,
+  type KnowledgeIngestFailedFileResult,
+  KnowledgeIngestFailedFileResultSchema,
   type KnowledgeIngestFileResult,
   KnowledgeIngestFileResultSchema,
+  type KnowledgeIngestJobResult,
+  KnowledgeIngestJobResultCountsSchema,
+  KnowledgeIngestJobResultMetadataSchema,
+  KnowledgeIngestJobResultSchema,
+  type KnowledgeIngestSkippedFileResult,
+  KnowledgeIngestSkippedFileResultSchema,
   PageInfoSchema,
   type PaginatedCronJobsResponse,
   PaginatedCronJobsResponseSchema,

package/src/src/jobs/schemas.ts

@@ -9,6 +9,7 @@ export const JobStatusSchema = z.enum([
 ]);
 
 export const CronJobStatusSchema = z.enum(["active", "paused", "deleting"]);
+export const JobKindSchema = z.enum(["knowledge_ingest"]).nullable();
 
 export const ReservedJobTargetFamilySchema = z.enum(["task:*", "workflow:*", "deploy:*"]);
 
@@ -20,6 +21,12 @@ export const PageInfoSchema = z.object({
 });
 
 const JsonObjectSchema = z.record(z.unknown());
+const KnowledgeIngestSkipReasonSchema = z.enum([
+  "hidden_path",
+  "ignored_directory",
+  "unsupported_file_type",
+]);
+const KnowledgeIngestFailureReasonSchema = z.enum(["parser_error", "upload_error"]);
 
 export const KnowledgeIngestFileResultSchema = z.object({
   source: z.string(),
@@ -33,27 +40,82 @@ export const KnowledgeIngestFileResultSchema = z.object({
   warnings: z.array(z.string()),
 });
 
+export const KnowledgeIngestSkippedFileResultSchema = z.object({
+  source: z.string(),
+  localSourcePath: z.string().nullable(),
+  reason: KnowledgeIngestSkipReasonSchema,
+  message: z.string(),
+});
+
+export const KnowledgeIngestFailedFileResultSchema = z.object({
+  source: z.string(),
+  localSourcePath: z.string(),
+  reason: KnowledgeIngestFailureReasonSchema,
+  message: z.string(),
+});
+
+export const KnowledgeIngestJobResultMetadataSchema = z.object({
+  requested_count: z.number().int().nonnegative(),
+  source_mode: z.enum(["explicit_sources", "path_prefix"]),
+  knowledge_path: z.string(),
+});
+
+export const KnowledgeIngestJobResultCountsSchema = z.object({
+  requested_count: z.number().int().nonnegative(),
+  ingested_count: z.number().int().nonnegative(),
+  skipped_count: z.number().int().nonnegative(),
+  failed_count: z.number().int().nonnegative(),
+});
+
+export const KnowledgeIngestJobResultSchema = z.object({
+  kind: z.literal("knowledge_ingest"),
+  version: z.literal(1),
+  metadata: KnowledgeIngestJobResultMetadataSchema,
+  summary: KnowledgeIngestJobResultCountsSchema,
+  ingested: z.array(KnowledgeIngestFileResultSchema),
+  skipped: z.array(KnowledgeIngestSkippedFileResultSchema),
+  failed: z.array(KnowledgeIngestFailedFileResultSchema),
+});
+
 export const JobResultSchema = z
+  .discriminatedUnion("kind", [
+    KnowledgeIngestJobResultSchema,
+    z.object({
+      kind: z.literal("value"),
+      value: z.unknown(),
+    }),
+    z.object({
+      kind: z.literal("artifacts"),
+      artifacts: z.array(z.unknown()),
+    }),
+  ])
+  .nullable();
+
+export const JobResultSummarySchema = z
   .discriminatedUnion("kind", [
     z.object({
       kind: z.literal("knowledge_ingest"),
-      files: z.array(KnowledgeIngestFileResultSchema),
+      state: z.enum(["success", "partial_success", "failed"]),
+      requested_count: z.number().int().nonnegative(),
+      ingested_count: z.number().int().nonnegative(),
+      skipped_count: z.number().int().nonnegative(),
+      failed_count: z.number().int().nonnegative(),
     }),
     z.object({
       kind: z.literal("value"),
-      value: z.unknown(),
     }),
     z.object({
       kind: z.literal("artifacts"),
-      artifacts: z.array(z.unknown()),
+      artifact_count: z.number().int().nonnegative(),
     }),
   ])
   .nullable();
 
-export const JobSchema = z.object({
+const BaseJobSchema = z.object({
   id: z.string().uuid(),
   project_id: z.string().uuid(),
   environment_id: z.string().uuid().nullable(),
+  branch_id: z.string().nullable().optional().default(null),
   cron_job_id: z.string().uuid().nullable(),
   batch_id: z.string().uuid().nullable(),
   name: z.string(),
@@ -64,9 +126,6 @@ export const JobSchema = z.object({
   timeout_seconds: z.number(),
   backoff_limit: z.number(),
   exit_code: z.number().nullable(),
-  failed_reason: z.string().nullable(),
-  failure_detail: z.string().nullable(),
-  result: JobResultSchema,
   started_at: z.string().nullable(),
   completed_at: z.string().nullable(),
   created_by: z.string().uuid().nullable(),
@@ -74,8 +133,22 @@ export const JobSchema = z.object({
   updated_at: z.string(),
 });
 
+export const JobSchema = BaseJobSchema.extend({
+  failed_reason: z.string().nullable().optional().default(null),
+  kind: JobKindSchema.optional().default(null),
+  failure_detail: z.string().nullable().optional().default(null),
+  result_summary: JobResultSummarySchema.optional().default(null),
+  result: JobResultSchema.optional().default(null),
+});
+
+export const JobListItemSchema = BaseJobSchema.extend({
+  kind: JobKindSchema.optional().default(null),
+  failure_detail: z.string().nullable().optional().default(null),
+  result_summary: JobResultSummarySchema.optional().default(null),
+});
+
 export const PaginatedJobsResponseSchema = z.object({
-  data: z.array(JobSchema),
+  data: z.array(JobListItemSchema),
   page_info: PageInfoSchema,
 });
 
@@ -111,6 +184,10 @@ export const KnowledgeIngestBatchSourceSchema = z.object({
   warning_count: z.number().int().nonnegative(),
 });
 
+export const KnowledgeIngestBatchSourceWithMessageSchema = KnowledgeIngestBatchSourceSchema.extend({
+  message: z.string(),
+});
+
 export const JobBatchStatusCountsSchema = z.object({
   submitted: z.number().int().nonnegative(),
   working: z.number().int().nonnegative(),
@@ -125,8 +202,12 @@ export const JobBatchResultSchema = z
       kind: z.literal("knowledge_ingest"),
       total_count: z.number().int().nonnegative(),
       completed_count: z.number().int().nonnegative(),
+      skipped_count: z.number().int().nonnegative(),
+      failed_count: z.number().int().nonnegative(),
       processing: z.array(KnowledgeIngestBatchSourceSchema),
       completed: z.array(KnowledgeIngestBatchSourceSchema),
+      skipped: z.array(KnowledgeIngestBatchSourceWithMessageSchema),
+      failed: z.array(KnowledgeIngestBatchSourceWithMessageSchema),
       remaining: z.array(KnowledgeIngestBatchSourceSchema),
       remaining_label: z.enum(["Remaining Files", "Not Ingested Files"]),
     }),
@@ -161,6 +242,7 @@ export const CronJobSchema = z.object({
   id: z.string().uuid(),
   project_id: z.string().uuid(),
   environment_id: z.string().uuid().nullable(),
+  branch_id: z.string().nullable().optional().default(null),
   name: z.string(),
   status: CronJobStatusSchema,
   target: z.string(),
@@ -184,17 +266,32 @@ export const PaginatedCronJobsResponseSchema = z.object({
 
 export type JobStatus = z.infer<typeof JobStatusSchema>;
 export type CronJobStatus = z.infer<typeof CronJobStatusSchema>;
+export type JobKind = z.infer<typeof JobKindSchema>;
 export type ReservedJobTargetFamily = z.infer<typeof ReservedJobTargetFamilySchema>;
 
 export type KnowledgeIngestFileResult = z.infer<typeof KnowledgeIngestFileResultSchema>;
+export type KnowledgeIngestSkippedFileResult = z.infer<
+  typeof KnowledgeIngestSkippedFileResultSchema
+>;
+export type KnowledgeIngestFailedFileResult = z.infer<typeof KnowledgeIngestFailedFileResultSchema>;
+export type KnowledgeIngestJobResultMetadata = z.infer<
+  typeof KnowledgeIngestJobResultMetadataSchema
+>;
+export type KnowledgeIngestJobResultCounts = z.infer<typeof KnowledgeIngestJobResultCountsSchema>;
+export type KnowledgeIngestJobResult = z.infer<typeof KnowledgeIngestJobResultSchema>;
 export type JobResult = z.infer<typeof JobResultSchema>;
+export type JobResultSummary = z.infer<typeof JobResultSummarySchema>;
 export type Job = z.infer<typeof JobSchema>;
+export type JobListItem = z.infer<typeof JobListItemSchema>;
 export type PaginatedJobsResponse = z.infer<typeof PaginatedJobsResponseSchema>;
 export type JobEvent = z.infer<typeof JobEventSchema>;
 export type JobEventsResponse = z.infer<typeof JobEventsResponseSchema>;
 export type JobLogsResponse = z.infer<typeof JobLogsResponseSchema>;
 
 export type KnowledgeIngestBatchSource = z.infer<typeof KnowledgeIngestBatchSourceSchema>;
+export type KnowledgeIngestBatchSourceWithMessage = z.infer<
+  typeof KnowledgeIngestBatchSourceWithMessageSchema
+>;
 export type JobBatchStatusCounts = z.infer<typeof JobBatchStatusCountsSchema>;
 export type JobBatchResult = z.infer<typeof JobBatchResultSchema>;
 export type JobBatch = z.infer<typeof JobBatchSchema>;

package/src/src/proxy/error-response.ts

@@ -0,0 +1,33 @@
+import * as dntShim from "../../_dnt.shims.js";
+import { ErrorPages } from "../server/utils/error-html.js";
+import type { ProxyContext } from "./handler.js";
+
+type ProxyError = NonNullable<ProxyContext["error"]>;
+
+export function jsonErrorResponse(status: number, body: Record<string, unknown>): dntShim.Response {
+  return new dntShim.Response(JSON.stringify(body), {
+    status,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+export function createProxyErrorResponse(error: ProxyError): dntShim.Response {
+  if (error.redirectUrl) {
+    return new dntShim.Response(null, {
+      status: 302,
+      headers: { Location: error.redirectUrl },
+    });
+  }
+
+  if (error.slug === "release-not-found" || error.slug === "project-not-found") {
+    return new dntShim.Response(ErrorPages.notFound(), {
+      status: 404,
+      headers: { "Content-Type": "text/html; charset=utf-8" },
+    });
+  }
+
+  return jsonErrorResponse(error.status, {
+    error: error.message,
+    status: error.status,
+  });
+}

package/src/src/proxy/handler.ts

@@ -175,6 +175,12 @@ function extractUserToken(cookieHeader: string): string | undefined {
   return match?.[1] ? decodeURIComponent(match[1]) : undefined;
 }
 
+function getStatusFromError(error: unknown): number | null {
+  const message = error instanceof Error ? error.message : String(error);
+  const match = message.match(/failed: (\d+)/);
+  return match ? Number(match[1]) : null;
+}
+
 async function extractUserIdFromToken(
   token: string,
   apiBaseUrl: string,
@@ -461,6 +467,7 @@ export function createProxyHandler(options: ProxyHandlerOptions) {
     const userToken = extractUserToken(cookieHeader);
 
     let token: string | undefined;
+    let tokenFetchError: unknown;
 
     if (isLocalProject) {
       logger?.debug("Local project, skipping token fetch", { localPath });
@@ -476,6 +483,7 @@ export function createProxyHandler(options: ProxyHandlerOptions) {
           try {
             token = await tokenManager.getToken(scope, projectSlug, customDomain);
           } catch (error) {
+            tokenFetchError = error;
             logger?.error("Token fetch failed", error as Error, { projectSlug, customDomain });
           }
         }
@@ -488,6 +496,29 @@ export function createProxyHandler(options: ProxyHandlerOptions) {
 
       const base = { scope, host, parsedDomain };
 
+      if (projectSlug && scope === "preview" && !token) {
+        const status = getStatusFromError(tokenFetchError);
+        if (status === 404) {
+          logger?.info("Preview project not found", { projectSlug, host });
+          return makeErrorContext(
+            base,
+            404,
+            "Preview project not found",
+            undefined,
+            undefined,
+            "project-not-found",
+          );
+        }
+
+        logger?.warn("Preview request has no usable token", {
+          projectSlug,
+          host,
+          hadUserToken: !!userToken,
+          hadTokenFetchError: !!tokenFetchError,
+        });
+        return makeErrorContext(base, 502, "Failed to authenticate preview request");
+      }
+
       if (isCustomDomain && !projectSlug) {
         if (!token) {
           logger?.error("Cannot process custom domain without token", undefined, { domain: host });
@@ -590,6 +621,18 @@ export function createProxyHandler(options: ProxyHandlerOptions) {
           );
         }
 
+        if (!resolved.projectId) {
+          logger?.info("Preview project not found after lookup", { projectSlug, host });
+          return makeErrorContext(
+            base,
+            404,
+            "Preview project not found",
+            token,
+            undefined,
+            "project-not-found",
+          );
+        }
+
         projectId = resolved.projectId;
         environmentId = resolved.environmentId;