veryfront 0.1.82 → 0.1.84

package/src/cli/commands/knowledge/parser-source.ts

@@ -14,6 +14,53 @@ def yaml_quote(value: Any) -> str:
 
 
 CODE_FENCE = chr(96) * 3
+TEXT_FILE_EXTENSIONS = {
+    ".c",
+    ".cc",
+    ".conf",
+    ".cpp",
+    ".cs",
+    ".css",
+    ".go",
+    ".h",
+    ".hpp",
+    ".ini",
+    ".java",
+    ".js",
+    ".jsonl",
+    ".jsx",
+    ".kt",
+    ".less",
+    ".lua",
+    ".mjs",
+    ".ndjson",
+    ".php",
+    ".pl",
+    ".py",
+    ".r",
+    ".rb",
+    ".rs",
+    ".sass",
+    ".scala",
+    ".scss",
+    ".sh",
+    ".sql",
+    ".swift",
+    ".toml",
+    ".ts",
+    ".tsx",
+    ".xml",
+    ".yaml",
+    ".yml",
+    ".zsh",
+}
+TEXT_FILE_NAMES = {
+    "dockerfile",
+    "makefile",
+    "readme",
+    "license",
+    "changelog",
+}
 
 
 def slugify(value: str) -> str:
@@ -406,6 +453,7 @@ def parse_json(path: str):
 
 def select_parser(path: Path):
     ext = path.suffix.lower()
+    name = path.name.lower()
     if ext == ".pdf":
         return "pdf", prefer_kreuzberg("pdf", parse_pdf)
     if ext in {".csv", ".tsv"}:
@@ -424,6 +472,10 @@ def select_parser(path: Path):
         return ext.lstrip("."), parse_text
     if ext == ".json":
         return "json", parse_json
+    if ext in TEXT_FILE_EXTENSIONS:
+        return ext.lstrip("."), parse_text
+    if not ext and name in TEXT_FILE_NAMES:
+        return "text", parse_text
     raise ValueError(f"Unsupported file type: {ext}")
 
 

package/src/cli/commands/knowledge/result.ts

@@ -0,0 +1,88 @@
+export type KnowledgeIngestSkipReason =
+  | "hidden_path"
+  | "ignored_directory"
+  | "unsupported_file_type";
+
+export type KnowledgeIngestFailureReason =
+  | "parser_error"
+  | "upload_error";
+
+export interface KnowledgeIngestFileResult {
+  source: string;
+  localSourcePath: string;
+  outputPath: string;
+  remotePath: string;
+  slug: string;
+  sourceType: string;
+  summary: string;
+  stats: Record<string, unknown>;
+  warnings: string[];
+}
+
+export interface KnowledgeIngestSkippedFileResult {
+  source: string;
+  localSourcePath: string | null;
+  reason: KnowledgeIngestSkipReason;
+  message: string;
+}
+
+export interface KnowledgeIngestFailedFileResult {
+  source: string;
+  localSourcePath: string;
+  reason: KnowledgeIngestFailureReason;
+  message: string;
+}
+
+export interface KnowledgeIngestSummary {
+  requested_count: number;
+  ingested_count: number;
+  skipped_count: number;
+  failed_count: number;
+}
+
+export interface KnowledgeIngestResultMetadata {
+  requested_count: number;
+  source_mode: "explicit_sources" | "path_prefix";
+  knowledge_path: string;
+}
+
+export interface KnowledgeIngestJobResult {
+  kind: "knowledge_ingest";
+  version: 1;
+  metadata: KnowledgeIngestResultMetadata;
+  summary: KnowledgeIngestSummary;
+  ingested: KnowledgeIngestFileResult[];
+  skipped: KnowledgeIngestSkippedFileResult[];
+  failed: KnowledgeIngestFailedFileResult[];
+}
+
+export function buildKnowledgeIngestJobResult(input: {
+  requestedCount: number;
+  sourceMode: KnowledgeIngestResultMetadata["source_mode"];
+  knowledgePath: string;
+  ingested: KnowledgeIngestFileResult[];
+  skipped?: KnowledgeIngestSkippedFileResult[];
+  failed?: KnowledgeIngestFailedFileResult[];
+}): KnowledgeIngestJobResult {
+  const skipped = input.skipped ?? [];
+  const failed = input.failed ?? [];
+
+  return {
+    kind: "knowledge_ingest",
+    version: 1,
+    metadata: {
+      requested_count: input.requestedCount,
+      source_mode: input.sourceMode,
+      knowledge_path: input.knowledgePath,
+    },
+    summary: {
+      requested_count: input.requestedCount,
+      ingested_count: input.ingested.length,
+      skipped_count: skipped.length,
+      failed_count: failed.length,
+    },
+    ingested: input.ingested,
+    skipped,
+    failed,
+  };
+}

package/src/cli/commands/knowledge/source-policy.ts

@@ -0,0 +1,164 @@
+import { basename, extname, normalize } from "../../../src/platform/compat/path/index.js";
+import type { KnowledgeIngestSkipReason } from "./result.js";
+
+const RICH_FILE_EXTENSIONS = new Set([
+  ".pdf",
+  ".csv",
+  ".tsv",
+  ".docx",
+  ".xlsx",
+  ".xls",
+  ".pptx",
+  ".html",
+  ".htm",
+  ".txt",
+  ".json",
+  ".md",
+  ".mdx",
+]);
+
+const TEXT_FALLBACK_EXTENSIONS = new Set([
+  ".c",
+  ".cc",
+  ".conf",
+  ".cpp",
+  ".cs",
+  ".css",
+  ".go",
+  ".h",
+  ".hpp",
+  ".ini",
+  ".java",
+  ".js",
+  ".jsonl",
+  ".jsx",
+  ".kt",
+  ".less",
+  ".lua",
+  ".mjs",
+  ".ndjson",
+  ".php",
+  ".pl",
+  ".py",
+  ".r",
+  ".rb",
+  ".rs",
+  ".sass",
+  ".scala",
+  ".scss",
+  ".sh",
+  ".sql",
+  ".swift",
+  ".toml",
+  ".tsx",
+  ".ts",
+  ".xml",
+  ".yaml",
+  ".yml",
+  ".zsh",
+]);
+
+const TEXT_FALLBACK_FILENAMES = new Set([
+  "dockerfile",
+  "makefile",
+  "readme",
+  "license",
+  "changelog",
+]);
+
+const IGNORED_DIRECTORY_NAMES = new Set([
+  "node_modules",
+]);
+
+export type KnowledgeIngestSourceDecision =
+  | { kind: "ingest" }
+  | {
+    kind: "skip";
+    reason: KnowledgeIngestSkipReason;
+    message: string;
+  };
+
+function normalizePathSegments(value: string): string[] {
+  return normalize(value)
+    .replaceAll("\\", "/")
+    .split("/")
+    .map((segment) => segment.trim())
+    .filter((segment) => segment.length > 0 && segment !== "." && segment !== "..");
+}
+
+function findHiddenSegment(segments: string[]): string | null {
+  for (const segment of segments) {
+    if (segment.startsWith(".")) {
+      return segment;
+    }
+  }
+
+  return null;
+}
+
+function findIgnoredDirectory(segments: string[]): string | null {
+  for (const segment of segments) {
+    if (IGNORED_DIRECTORY_NAMES.has(segment.toLowerCase())) {
+      return segment;
+    }
+  }
+
+  return null;
+}
+
+function isSupportedKnowledgeFile(path: string): boolean {
+  const extension = extname(path).toLowerCase();
+  if (
+    extension && (RICH_FILE_EXTENSIONS.has(extension) || TEXT_FALLBACK_EXTENSIONS.has(extension))
+  ) {
+    return true;
+  }
+
+  const fileName = basename(path).toLowerCase();
+  return TEXT_FALLBACK_FILENAMES.has(fileName);
+}
+
+function classifyCommonKnowledgePath(path: string): KnowledgeIngestSourceDecision | null {
+  const segments = normalizePathSegments(path);
+  const hiddenSegment = findHiddenSegment(segments);
+  if (hiddenSegment != null) {
+    return {
+      kind: "skip",
+      reason: "hidden_path",
+      message: `Hidden file or directory skipped: ${hiddenSegment}`,
+    };
+  }
+
+  const ignoredDirectory = findIgnoredDirectory(segments);
+  if (ignoredDirectory != null) {
+    return {
+      kind: "skip",
+      reason: "ignored_directory",
+      message: `Ignored directory skipped: ${ignoredDirectory}`,
+    };
+  }
+
+  return null;
+}
+
+export function classifyKnowledgeDirectoryPath(path: string): KnowledgeIngestSourceDecision {
+  return classifyCommonKnowledgePath(path) ?? { kind: "ingest" };
+}
+
+export function classifyKnowledgeSourcePath(path: string): KnowledgeIngestSourceDecision {
+  const commonDecision = classifyCommonKnowledgePath(path);
+  if (commonDecision != null) {
+    return commonDecision;
+  }
+
+  if (!isSupportedKnowledgeFile(path)) {
+    const extension = extname(path).toLowerCase();
+    return {
+      kind: "skip",
+      reason: "unsupported_file_type",
+      message: `Unsupported file type: ${extension || basename(path)}`,
+    };
+  }
+
+  return { kind: "ingest" };
+}

package/src/deno.js

@@ -1,6 +1,6 @@
 export default {
   "name": "veryfront",
-  "version": "0.1.82",
+  "version": "0.1.84",
   "license": "Apache-2.0",
   "nodeModulesDir": "auto",
   "exclude": [

package/src/src/jobs/index.ts

@@ -60,10 +60,16 @@ export {
   JobEventSchema,
   type JobEventsResponse,
   JobEventsResponseSchema,
+  type JobKind,
+  JobKindSchema,
+  type JobListItem,
+  JobListItemSchema,
   type JobLogsResponse,
   JobLogsResponseSchema,
   type JobResult,
   JobResultSchema,
+  type JobResultSummary,
+  JobResultSummarySchema,
   JobSchema,
   type JobStatus,
   JobStatusSchema,
@@ -73,8 +79,18 @@ export {
   JobTargetDefinitionsResponseSchema,
   type KnowledgeIngestBatchSource,
   KnowledgeIngestBatchSourceSchema,
+  type KnowledgeIngestBatchSourceWithMessage,
+  KnowledgeIngestBatchSourceWithMessageSchema,
+  type KnowledgeIngestFailedFileResult,
+  KnowledgeIngestFailedFileResultSchema,
   type KnowledgeIngestFileResult,
   KnowledgeIngestFileResultSchema,
+  type KnowledgeIngestJobResult,
+  KnowledgeIngestJobResultCountsSchema,
+  KnowledgeIngestJobResultMetadataSchema,
+  KnowledgeIngestJobResultSchema,
+  type KnowledgeIngestSkippedFileResult,
+  KnowledgeIngestSkippedFileResultSchema,
   PageInfoSchema,
   type PaginatedCronJobsResponse,
   PaginatedCronJobsResponseSchema,

package/src/src/jobs/schemas.ts

@@ -9,6 +9,7 @@ export const JobStatusSchema = z.enum([
 ]);
 
 export const CronJobStatusSchema = z.enum(["active", "paused", "deleting"]);
+export const JobKindSchema = z.enum(["knowledge_ingest"]).nullable();
 
 export const ReservedJobTargetFamilySchema = z.enum(["task:*", "workflow:*", "deploy:*"]);
 
@@ -20,6 +21,12 @@ export const PageInfoSchema = z.object({
 });
 
 const JsonObjectSchema = z.record(z.unknown());
+const KnowledgeIngestSkipReasonSchema = z.enum([
+  "hidden_path",
+  "ignored_directory",
+  "unsupported_file_type",
+]);
+const KnowledgeIngestFailureReasonSchema = z.enum(["parser_error", "upload_error"]);
 
 export const KnowledgeIngestFileResultSchema = z.object({
   source: z.string(),
@@ -33,27 +40,82 @@ export const KnowledgeIngestFileResultSchema = z.object({
   warnings: z.array(z.string()),
 });
 
+export const KnowledgeIngestSkippedFileResultSchema = z.object({
+  source: z.string(),
+  localSourcePath: z.string().nullable(),
+  reason: KnowledgeIngestSkipReasonSchema,
+  message: z.string(),
+});
+
+export const KnowledgeIngestFailedFileResultSchema = z.object({
+  source: z.string(),
+  localSourcePath: z.string(),
+  reason: KnowledgeIngestFailureReasonSchema,
+  message: z.string(),
+});
+
+export const KnowledgeIngestJobResultMetadataSchema = z.object({
+  requested_count: z.number().int().nonnegative(),
+  source_mode: z.enum(["explicit_sources", "path_prefix"]),
+  knowledge_path: z.string(),
+});
+
+export const KnowledgeIngestJobResultCountsSchema = z.object({
+  requested_count: z.number().int().nonnegative(),
+  ingested_count: z.number().int().nonnegative(),
+  skipped_count: z.number().int().nonnegative(),
+  failed_count: z.number().int().nonnegative(),
+});
+
+export const KnowledgeIngestJobResultSchema = z.object({
+  kind: z.literal("knowledge_ingest"),
+  version: z.literal(1),
+  metadata: KnowledgeIngestJobResultMetadataSchema,
+  summary: KnowledgeIngestJobResultCountsSchema,
+  ingested: z.array(KnowledgeIngestFileResultSchema),
+  skipped: z.array(KnowledgeIngestSkippedFileResultSchema),
+  failed: z.array(KnowledgeIngestFailedFileResultSchema),
+});
+
 export const JobResultSchema = z
+  .discriminatedUnion("kind", [
+    KnowledgeIngestJobResultSchema,
+    z.object({
+      kind: z.literal("value"),
+      value: z.unknown(),
+    }),
+    z.object({
+      kind: z.literal("artifacts"),
+      artifacts: z.array(z.unknown()),
+    }),
+  ])
+  .nullable();
+
+export const JobResultSummarySchema = z
   .discriminatedUnion("kind", [
     z.object({
       kind: z.literal("knowledge_ingest"),
-      files: z.array(KnowledgeIngestFileResultSchema),
+      state: z.enum(["success", "partial_success", "failed"]),
+      requested_count: z.number().int().nonnegative(),
+      ingested_count: z.number().int().nonnegative(),
+      skipped_count: z.number().int().nonnegative(),
+      failed_count: z.number().int().nonnegative(),
     }),
     z.object({
       kind: z.literal("value"),
-      value: z.unknown(),
     }),
     z.object({
       kind: z.literal("artifacts"),
-      artifacts: z.array(z.unknown()),
+      artifact_count: z.number().int().nonnegative(),
     }),
   ])
   .nullable();
 
-export const JobSchema = z.object({
+const BaseJobSchema = z.object({
   id: z.string().uuid(),
   project_id: z.string().uuid(),
   environment_id: z.string().uuid().nullable(),
+  branch_id: z.string().nullable().optional().default(null),
   cron_job_id: z.string().uuid().nullable(),
   batch_id: z.string().uuid().nullable(),
   name: z.string(),
@@ -64,9 +126,6 @@ export const JobSchema = z.object({
   timeout_seconds: z.number(),
   backoff_limit: z.number(),
   exit_code: z.number().nullable(),
-  failed_reason: z.string().nullable(),
-  failure_detail: z.string().nullable(),
-  result: JobResultSchema,
   started_at: z.string().nullable(),
   completed_at: z.string().nullable(),
   created_by: z.string().uuid().nullable(),
@@ -74,8 +133,22 @@ export const JobSchema = z.object({
   updated_at: z.string(),
 });
 
+export const JobSchema = BaseJobSchema.extend({
+  failed_reason: z.string().nullable().optional().default(null),
+  kind: JobKindSchema.optional().default(null),
+  failure_detail: z.string().nullable().optional().default(null),
+  result_summary: JobResultSummarySchema.optional().default(null),
+  result: JobResultSchema.optional().default(null),
+});
+
+export const JobListItemSchema = BaseJobSchema.extend({
+  kind: JobKindSchema.optional().default(null),
+  failure_detail: z.string().nullable().optional().default(null),
+  result_summary: JobResultSummarySchema.optional().default(null),
+});
+
 export const PaginatedJobsResponseSchema = z.object({
-  data: z.array(JobSchema),
+  data: z.array(JobListItemSchema),
   page_info: PageInfoSchema,
 });
 
@@ -111,6 +184,10 @@ export const KnowledgeIngestBatchSourceSchema = z.object({
   warning_count: z.number().int().nonnegative(),
 });
 
+export const KnowledgeIngestBatchSourceWithMessageSchema = KnowledgeIngestBatchSourceSchema.extend({
+  message: z.string(),
+});
+
 export const JobBatchStatusCountsSchema = z.object({
   submitted: z.number().int().nonnegative(),
   working: z.number().int().nonnegative(),
@@ -125,8 +202,12 @@ export const JobBatchResultSchema = z
       kind: z.literal("knowledge_ingest"),
       total_count: z.number().int().nonnegative(),
       completed_count: z.number().int().nonnegative(),
+      skipped_count: z.number().int().nonnegative(),
+      failed_count: z.number().int().nonnegative(),
       processing: z.array(KnowledgeIngestBatchSourceSchema),
       completed: z.array(KnowledgeIngestBatchSourceSchema),
+      skipped: z.array(KnowledgeIngestBatchSourceWithMessageSchema),
+      failed: z.array(KnowledgeIngestBatchSourceWithMessageSchema),
       remaining: z.array(KnowledgeIngestBatchSourceSchema),
       remaining_label: z.enum(["Remaining Files", "Not Ingested Files"]),
     }),
@@ -161,6 +242,7 @@ export const CronJobSchema = z.object({
   id: z.string().uuid(),
   project_id: z.string().uuid(),
   environment_id: z.string().uuid().nullable(),
+  branch_id: z.string().nullable().optional().default(null),
   name: z.string(),
   status: CronJobStatusSchema,
   target: z.string(),
@@ -184,17 +266,32 @@ export const PaginatedCronJobsResponseSchema = z.object({
 
 export type JobStatus = z.infer<typeof JobStatusSchema>;
 export type CronJobStatus = z.infer<typeof CronJobStatusSchema>;
+export type JobKind = z.infer<typeof JobKindSchema>;
 export type ReservedJobTargetFamily = z.infer<typeof ReservedJobTargetFamilySchema>;
 
 export type KnowledgeIngestFileResult = z.infer<typeof KnowledgeIngestFileResultSchema>;
+export type KnowledgeIngestSkippedFileResult = z.infer<
+  typeof KnowledgeIngestSkippedFileResultSchema
+>;
+export type KnowledgeIngestFailedFileResult = z.infer<typeof KnowledgeIngestFailedFileResultSchema>;
+export type KnowledgeIngestJobResultMetadata = z.infer<
+  typeof KnowledgeIngestJobResultMetadataSchema
+>;
+export type KnowledgeIngestJobResultCounts = z.infer<typeof KnowledgeIngestJobResultCountsSchema>;
+export type KnowledgeIngestJobResult = z.infer<typeof KnowledgeIngestJobResultSchema>;
 export type JobResult = z.infer<typeof JobResultSchema>;
+export type JobResultSummary = z.infer<typeof JobResultSummarySchema>;
 export type Job = z.infer<typeof JobSchema>;
+export type JobListItem = z.infer<typeof JobListItemSchema>;
 export type PaginatedJobsResponse = z.infer<typeof PaginatedJobsResponseSchema>;
 export type JobEvent = z.infer<typeof JobEventSchema>;
 export type JobEventsResponse = z.infer<typeof JobEventsResponseSchema>;
 export type JobLogsResponse = z.infer<typeof JobLogsResponseSchema>;
 
 export type KnowledgeIngestBatchSource = z.infer<typeof KnowledgeIngestBatchSourceSchema>;
+export type KnowledgeIngestBatchSourceWithMessage = z.infer<
+  typeof KnowledgeIngestBatchSourceWithMessageSchema
+>;
 export type JobBatchStatusCounts = z.infer<typeof JobBatchStatusCountsSchema>;
 export type JobBatchResult = z.infer<typeof JobBatchResultSchema>;
 export type JobBatch = z.infer<typeof JobBatchSchema>;

package/src/src/platform/adapters/base.ts

@@ -117,7 +117,11 @@ export interface FileSystemAdapter {
   makeTempDir(prefix: string): Promise<string>;
   watch(paths: string | string[], options?: WatchOptions): FileWatcher;
   /** Resolve a file path with extension fallback (e.g., pages/test → pages/test.mdx) */
-  resolveFile?(basePath: string): Promise<string | null>;
+  resolveFile?(basePath: string, options?: ResolveFileOptions): Promise<string | null>;
+}
+
+export interface ResolveFileOptions {
+  allowPagesPrefix?: boolean;
 }
 
 export interface DirEntry {

package/src/src/platform/adapters/fs/github/adapter.ts

@@ -1,6 +1,7 @@
 import { logger } from "../../../../utils/index.js";
 import { CONFIG_INVALID } from "../../../../errors/index.js";
 import { getEnv } from "../../../compat/process.js";
+import type { ResolveFileOptions } from "../../base.js";
 import { FileCache } from "../cache/file-cache.js";
 import type { FSAdapter, FSAdapterConfig } from "../veryfront/types.js";
 import { GitHubApiClient } from "./github-api-client.js";
@@ -115,9 +116,9 @@ export class GitHubFSAdapter implements FSAdapter {
     return this.dirOps.readdir(path);
   }
 
-  async resolveFile(basePath: string): Promise<string | null> {
+  async resolveFile(basePath: string, options?: ResolveFileOptions): Promise<string | null> {
     await this.ensureInitialized();
-    return this.statOps.resolveFile(basePath);
+    return this.statOps.resolveFile(basePath, options);
   }
 
   getCacheStats(): {

package/src/src/platform/adapters/fs/github/stat-operations.ts

@@ -5,6 +5,7 @@ import {
   buildGitHubStatCacheKey,
   buildGitHubTreeCacheKey,
 } from "../../../../cache/index.js";
+import type { ResolveFileOptions } from "../../base.js";
 import type { FileCache } from "../cache/file-cache.js";
 import type { GitHubApiClient } from "./github-api-client.js";
 import type { FileIndexEntry, FileInfo, GitHubTreeEntry, ResolvedGitHubConfig } from "./types.js";
@@ -176,7 +177,7 @@ export class GitHubStatOperations {
     }
   }
 
-  async resolveFile(basePath: string): Promise<string | null> {
+  async resolveFile(basePath: string, options?: ResolveFileOptions): Promise<string | null> {
     await this.ensureIndex();
 
     const normalizedPath = normalizeGitHubPath(basePath, this.projectDir);
@@ -185,7 +186,7 @@ export class GitHubStatOperations {
     if (cached !== undefined) return cached;
 
     const resolved = this.tryResolve(normalizedPath) ??
-      this.tryResolveWithPagesPrefix(normalizedPath);
+      (options?.allowPagesPrefix === false ? null : this.tryResolveWithPagesPrefix(normalizedPath));
 
     this.cache.set(cacheKey, resolved);
     return resolved;

package/src/src/platform/adapters/fs/veryfront/adapter.ts

@@ -9,7 +9,7 @@ import type {
   InvalidationCallbacks,
   ResolvedContentContext,
 } from "./types.js";
-import type { FileInfo } from "../../base.js";
+import type { FileInfo, ResolveFileOptions } from "../../base.js";
 import { VeryfrontApiClient } from "../../veryfront-api-client/index.js";
 import type { Project } from "../../veryfront-api-client/index.js";
 import { FileCache } from "../cache/file-cache.js";
@@ -142,6 +142,24 @@ export class VeryfrontFSAdapter implements FSAdapter {
 
         return result;
       },
+      hasCachedFileList: async () => {
+        if (!this.contentContext) {
+          logger.debug("hasCachedFileList: no contentContext");
+          return false;
+        }
+
+        const cacheKey = buildFileListCacheKey(this.contentContext);
+        const result = await this.cache.getAsync<Array<{ path: string }>>(cacheKey);
+        const hasResult = Array.isArray(result) && result.length > 0;
+
+        logger.debug("hasCachedFileList lookup", {
+          cacheKey,
+          hasResult,
+          resultSize: result?.length ?? 0,
+        });
+
+        return hasResult;
+      },
       isPersistentCacheInvalidated: (prefix: string) => this.isPersistentCacheInvalidated(prefix),
       isReleaseBeingInvalidated: (releaseId: string) =>
         this.isPersistentCacheInvalidated(
@@ -410,9 +428,12 @@ export class VeryfrontFSAdapter implements FSAdapter {
     return this.statOps.exists(path);
   }
 
-  async resolveFile(basePath: string): Promise<string | null> {
+  async resolveFile(
+    basePath: string,
+    options?: ResolveFileOptions,
+  ): Promise<string | null> {
     await this.ensureInitialized();
-    return this.statOps.resolveFile(basePath);
+    return this.statOps.resolveFile(basePath, options);
   }
 
   dispose(): void {