stagent 0.1.0

package/src/lib/db/schema.ts

@@ -0,0 +1,227 @@
+import { sqliteTable, text, integer, index } from "drizzle-orm/sqlite-core";
+import type { InferSelectModel } from "drizzle-orm";
+
+export const projects = sqliteTable("projects", {
+  id: text("id").primaryKey(),
+  name: text("name").notNull(),
+  description: text("description"),
+  workingDirectory: text("working_directory"),
+  status: text("status", { enum: ["active", "paused", "completed"] })
+    .default("active")
+    .notNull(),
+  createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+  updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+});
+
+export const tasks = sqliteTable(
+  "tasks",
+  {
+    id: text("id").primaryKey(),
+    projectId: text("project_id").references(() => projects.id),
+    workflowId: text("workflow_id").references(() => workflows.id),
+    scheduleId: text("schedule_id").references(() => schedules.id),
+    title: text("title").notNull(),
+    description: text("description"),
+    status: text("status", {
+      enum: ["planned", "queued", "running", "completed", "failed", "cancelled"],
+    })
+      .default("planned")
+      .notNull(),
+    assignedAgent: text("assigned_agent"),
+    agentProfile: text("agent_profile"),
+    priority: integer("priority").default(2).notNull(),
+    result: text("result"),
+    sessionId: text("session_id"),
+    resumeCount: integer("resume_count").default(0).notNull(),
+    createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+    updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_tasks_status").on(table.status),
+    index("idx_tasks_project_id").on(table.projectId),
+    index("idx_tasks_workflow_id").on(table.workflowId),
+    index("idx_tasks_schedule_id").on(table.scheduleId),
+    index("idx_tasks_agent_profile").on(table.agentProfile),
+  ]
+);
+
+export const workflows = sqliteTable("workflows", {
+  id: text("id").primaryKey(),
+  projectId: text("project_id").references(() => projects.id),
+  name: text("name").notNull(),
+  definition: text("definition").notNull(),
+  status: text("status", {
+    enum: ["draft", "active", "paused", "completed", "failed"],
+  })
+    .default("draft")
+    .notNull(),
+  createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+  updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+});
+
+export const agentLogs = sqliteTable(
+  "agent_logs",
+  {
+    id: text("id").primaryKey(),
+    taskId: text("task_id").references(() => tasks.id),
+    agentType: text("agent_type").notNull(),
+    event: text("event").notNull(),
+    payload: text("payload"),
+    timestamp: integer("timestamp", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_agent_logs_task_id").on(table.taskId),
+    index("idx_agent_logs_timestamp").on(table.timestamp),
+  ]
+);
+
+export const notifications = sqliteTable(
+  "notifications",
+  {
+    id: text("id").primaryKey(),
+    taskId: text("task_id").references(() => tasks.id),
+    type: text("type", {
+      enum: [
+        "permission_required",
+        "task_completed",
+        "task_failed",
+        "agent_message",
+        "budget_alert",
+      ],
+    }).notNull(),
+    title: text("title").notNull(),
+    body: text("body"),
+    read: integer("read", { mode: "boolean" }).default(false).notNull(),
+    toolName: text("tool_name"),
+    toolInput: text("tool_input"),
+    response: text("response"),
+    respondedAt: integer("responded_at", { mode: "timestamp" }),
+    createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_notifications_task_id").on(table.taskId),
+    index("idx_notifications_read").on(table.read),
+  ]
+);
+
+export const documents = sqliteTable(
+  "documents",
+  {
+    id: text("id").primaryKey(),
+    taskId: text("task_id").references(() => tasks.id),
+    projectId: text("project_id").references(() => projects.id),
+    filename: text("filename").notNull(),
+    originalName: text("original_name").notNull(),
+    mimeType: text("mime_type").notNull(),
+    size: integer("size").notNull(),
+    storagePath: text("storage_path").notNull(),
+    version: integer("version").default(1).notNull(),
+    direction: text("direction", { enum: ["input", "output"] })
+      .default("input")
+      .notNull(),
+    category: text("category"),
+    status: text("status", {
+      enum: ["uploaded", "processing", "ready", "error"],
+    })
+      .default("uploaded")
+      .notNull(),
+    extractedText: text("extracted_text"),
+    processedPath: text("processed_path"),
+    processingError: text("processing_error"),
+    createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+    updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_documents_task_id").on(table.taskId),
+    index("idx_documents_project_id").on(table.projectId),
+  ]
+);
+
+export const schedules = sqliteTable(
+  "schedules",
+  {
+    id: text("id").primaryKey(),
+    projectId: text("project_id").references(() => projects.id),
+    name: text("name").notNull(),
+    prompt: text("prompt").notNull(),
+    cronExpression: text("cron_expression").notNull(),
+    assignedAgent: text("assigned_agent"),
+    agentProfile: text("agent_profile"),
+    recurs: integer("recurs", { mode: "boolean" }).default(true).notNull(),
+    status: text("status", {
+      enum: ["active", "paused", "completed", "expired"],
+    })
+      .default("active")
+      .notNull(),
+    maxFirings: integer("max_firings"),
+    firingCount: integer("firing_count").default(0).notNull(),
+    expiresAt: integer("expires_at", { mode: "timestamp" }),
+    lastFiredAt: integer("last_fired_at", { mode: "timestamp" }),
+    nextFireAt: integer("next_fire_at", { mode: "timestamp" }),
+    createdAt: integer("created_at", { mode: "timestamp" }).notNull(),
+    updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_schedules_status").on(table.status),
+    index("idx_schedules_next_fire_at").on(table.nextFireAt),
+    index("idx_schedules_project_id").on(table.projectId),
+  ]
+);
+
+export const settings = sqliteTable("settings", {
+  key: text("key").primaryKey(),
+  value: text("value").notNull(),
+  updatedAt: integer("updated_at", { mode: "timestamp" }).notNull(),
+});
+
+export const usageLedger = sqliteTable(
+  "usage_ledger",
+  {
+    id: text("id").primaryKey(),
+    taskId: text("task_id").references(() => tasks.id),
+    workflowId: text("workflow_id").references(() => workflows.id),
+    scheduleId: text("schedule_id").references(() => schedules.id),
+    projectId: text("project_id").references(() => projects.id),
+    activityType: text("activity_type", {
+      enum: [
+        "task_run",
+        "task_resume",
+        "workflow_step",
+        "scheduled_firing",
+        "task_assist",
+        "profile_test",
+      ],
+    }).notNull(),
+    runtimeId: text("runtime_id").notNull(),
+    providerId: text("provider_id").notNull(),
+    modelId: text("model_id"),
+    status: text("status", {
+      enum: ["completed", "failed", "cancelled", "blocked", "unknown_pricing"],
+    }).notNull(),
+    inputTokens: integer("input_tokens"),
+    outputTokens: integer("output_tokens"),
+    totalTokens: integer("total_tokens"),
+    costMicros: integer("cost_micros"),
+    pricingVersion: text("pricing_version"),
+    startedAt: integer("started_at", { mode: "timestamp" }).notNull(),
+    finishedAt: integer("finished_at", { mode: "timestamp" }).notNull(),
+  },
+  (table) => [
+    index("idx_usage_ledger_task_id").on(table.taskId),
+    index("idx_usage_ledger_activity_type").on(table.activityType),
+    index("idx_usage_ledger_runtime_id").on(table.runtimeId),
+    index("idx_usage_ledger_provider_model").on(table.providerId, table.modelId),
+    index("idx_usage_ledger_finished_at").on(table.finishedAt),
+  ]
+);
+
+// Shared types derived from schema — use these in components instead of `as any`
+export type ProjectRow = InferSelectModel<typeof projects>;
+export type TaskRow = InferSelectModel<typeof tasks>;
+export type WorkflowRow = InferSelectModel<typeof workflows>;
+export type AgentLogRow = InferSelectModel<typeof agentLogs>;
+export type NotificationRow = InferSelectModel<typeof notifications>;
+export type DocumentRow = InferSelectModel<typeof documents>;
+export type ScheduleRow = InferSelectModel<typeof schedules>;
+export type SettingsRow = InferSelectModel<typeof settings>;
+export type UsageLedgerRow = InferSelectModel<typeof usageLedger>;

package/src/lib/documents/cleanup.ts

@@ -0,0 +1,50 @@
+import { readdir, stat, unlink } from "fs/promises";
+import { join } from "path";
+import { db } from "@/lib/db";
+import { documents } from "@/lib/db/schema";
+import { getStagentUploadsDir } from "@/lib/utils/stagent-paths";
+import { eq } from "drizzle-orm";
+
+const UPLOAD_DIR = getStagentUploadsDir();
+const ORPHAN_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours
+
+export async function cleanupOrphanedUploads(): Promise<{
+  deleted: string[];
+  errors: string[];
+}> {
+  const deleted: string[] = [];
+  const errors: string[] = [];
+
+  try {
+    const files = await readdir(UPLOAD_DIR);
+    const now = Date.now();
+
+    for (const filename of files) {
+      const fileId = filename.split(".")[0];
+      const filepath = join(UPLOAD_DIR, filename);
+
+      try {
+        const fileStat = await stat(filepath);
+        const ageMs = now - fileStat.mtimeMs;
+
+        if (ageMs < ORPHAN_AGE_MS) continue;
+
+        const [doc] = await db
+          .select()
+          .from(documents)
+          .where(eq(documents.id, fileId));
+
+        if (!doc) {
+          await unlink(filepath);
+          deleted.push(filename);
+        }
+      } catch (err) {
+        errors.push(`${filename}: ${err instanceof Error ? err.message : "unknown error"}`);
+      }
+    }
+  } catch {
+    // Upload directory may not exist yet
+  }
+
+  return { deleted, errors };
+}

package/src/lib/documents/context-builder.ts

@@ -0,0 +1,75 @@
+/**
+ * Build document context section for agent prompts.
+ * Queries documents linked to a task and formats them for the agent.
+ */
+
+import { db } from "@/lib/db";
+import { documents } from "@/lib/db/schema";
+import { and, eq } from "drizzle-orm";
+import type { DocumentRow } from "@/lib/db/schema";
+
+const MAX_INLINE_TEXT = 10_000;
+
+function formatDocument(doc: DocumentRow, index: number): string {
+  const header = `[Document ${index + 1}: ${doc.originalName}]`;
+  const pathLine = `Path: ${doc.storagePath}`;
+
+  const isImage = doc.mimeType.startsWith("image/");
+
+  // Images: path reference only (agent uses Read tool to view)
+  if (isImage) {
+    const meta = doc.extractedText ? `\n${doc.extractedText}` : "";
+    return `${header}\n${pathLine}\nType: ${doc.mimeType} (use Read tool to view)${meta}`;
+  }
+
+  // Processing or failed: path + status note
+  if (doc.status === "processing") {
+    return `${header}\n${pathLine}\nStatus: still processing — content not yet available`;
+  }
+
+  if (doc.status === "error") {
+    return `${header}\n${pathLine}\nStatus: processing failed (${doc.processingError ?? "unknown error"})`;
+  }
+
+  if (doc.status === "uploaded") {
+    return `${header}\n${pathLine}\nStatus: not yet processed`;
+  }
+
+  // Ready with extracted text
+  if (doc.extractedText) {
+    if (doc.extractedText.length < MAX_INLINE_TEXT) {
+      return `${header}\n${pathLine}\nContent:\n<document>\n${doc.extractedText}\n</document>`;
+    }
+    // Large document: truncated + path reference
+    const truncated = doc.extractedText.slice(0, MAX_INLINE_TEXT);
+    return `${header}\n${pathLine}\nContent (truncated to ${MAX_INLINE_TEXT} chars — use Read tool for full content):\n<document>\n${truncated}\n</document>`;
+  }
+
+  // Ready but no extracted text (unsupported format)
+  return `${header}\n${pathLine}\nType: ${doc.mimeType} (use Read tool to access)`;
+}
+
+/**
+ * Build the document context string for a task's prompt.
+ * Returns null if the task has no documents.
+ */
+export async function buildDocumentContext(
+  taskId: string
+): Promise<string | null> {
+  const docs = await db
+    .select()
+    .from(documents)
+    .where(and(eq(documents.taskId, taskId), eq(documents.direction, "input")));
+
+  if (docs.length === 0) return null;
+
+  const sections = docs.map((doc, i) => formatDocument(doc, i));
+
+  return [
+    "--- Attached Documents ---",
+    "",
+    ...sections,
+    "",
+    "--- End Attached Documents ---",
+  ].join("\n");
+}

package/src/lib/documents/output-scanner.ts

@@ -0,0 +1,166 @@
+import { homedir } from "node:os";
+import path from "node:path";
+import fs from "node:fs/promises";
+import { db } from "@/lib/db";
+import { documents, tasks } from "@/lib/db/schema";
+import { and, eq } from "drizzle-orm";
+import { processDocument } from "./processor";
+
+const STAGENT_DATA_DIR =
+  process.env.STAGENT_DATA_DIR || path.join(homedir(), ".stagent");
+const TASK_OUTPUTS_DIR = path.join(STAGENT_DATA_DIR, "outputs");
+const OUTPUT_ARCHIVE_DIR = path.join(STAGENT_DATA_DIR, "documents", "output");
+
+const OUTPUT_MIME_TYPES: Record<string, string> = {
+  ".md": "text/markdown",
+  ".txt": "text/plain",
+  ".json": "application/json",
+  ".csv": "text/csv",
+  ".html": "text/html",
+};
+
+export function getTaskOutputDirectory(taskId: string): string {
+  return path.join(TASK_OUTPUTS_DIR, taskId);
+}
+
+export async function prepareTaskOutputDirectory(
+  taskId: string,
+  options: { clearExisting?: boolean } = {}
+): Promise<string> {
+  const outputDir = getTaskOutputDirectory(taskId);
+
+  if (options.clearExisting) {
+    await fs.rm(outputDir, { recursive: true, force: true });
+  }
+
+  await fs.mkdir(outputDir, { recursive: true });
+  return outputDir;
+}
+
+export function buildTaskOutputInstructions(taskId: string): string {
+  const outputDir = getTaskOutputDirectory(taskId);
+  return [
+    "Generated file outputs:",
+    `- Write any final files to ${outputDir}`,
+    "- Files in .md, .json, .csv, .txt, and .html are automatically captured after completion",
+    "- Keep the final filename stable if you want Stagent to version rerun outputs cleanly",
+  ].join("\n");
+}
+
+async function listFilesRecursively(rootDir: string): Promise<string[]> {
+  const entries = await fs.readdir(rootDir, { withFileTypes: true });
+  const files = await Promise.all(
+    entries.map(async (entry) => {
+      const resolved = path.join(rootDir, entry.name);
+      if (entry.isDirectory()) {
+        return listFilesRecursively(resolved);
+      }
+      return [resolved];
+    })
+  );
+
+  return files.flat();
+}
+
+function resolveOutputMimeType(filename: string): string | null {
+  return OUTPUT_MIME_TYPES[path.extname(filename).toLowerCase()] ?? null;
+}
+
+function normalizeRelativePath(value: string): string {
+  return value.split(path.sep).join("/");
+}
+
+function buildArchivedFilename(relativePath: string, version: number): string {
+  const parsed = path.parse(relativePath);
+  const sanitizedBase = parsed.name.replace(/[^a-zA-Z0-9._-]+/g, "-");
+  const nestedPrefix = parsed.dir
+    ? `${parsed.dir.replace(/[\\/]+/g, "__")}__`
+    : "";
+  return `${nestedPrefix}${sanitizedBase || "output"}-v${version}${parsed.ext}`;
+}
+
+export async function scanTaskOutputDocuments(taskId: string): Promise<string[]> {
+  const outputDir = getTaskOutputDirectory(taskId);
+  const [task] = await db
+    .select({ id: tasks.id, projectId: tasks.projectId })
+    .from(tasks)
+    .where(eq(tasks.id, taskId));
+
+  if (!task) {
+    throw new Error(`Task ${taskId} not found`);
+  }
+
+  try {
+    await fs.access(outputDir);
+  } catch {
+    return [];
+  }
+
+  const existingOutputDocs = await db
+    .select({
+      originalName: documents.originalName,
+      version: documents.version,
+    })
+    .from(documents)
+    .where(
+      and(eq(documents.taskId, taskId), eq(documents.direction, "output"))
+    );
+
+  const versionMap = new Map<string, number>();
+  existingOutputDocs.forEach((doc) => {
+    versionMap.set(
+      doc.originalName,
+      Math.max(versionMap.get(doc.originalName) ?? 0, doc.version)
+    );
+  });
+
+  const discoveredFiles = await listFilesRecursively(outputDir);
+  const registeredDocumentIds: string[] = [];
+
+  for (const sourcePath of discoveredFiles) {
+    const relativePath = normalizeRelativePath(path.relative(outputDir, sourcePath));
+    const mimeType = resolveOutputMimeType(relativePath);
+    if (!mimeType) {
+      continue;
+    }
+
+    const stats = await fs.stat(sourcePath);
+    if (!stats.isFile()) {
+      continue;
+    }
+
+    const nextVersion = (versionMap.get(relativePath) ?? 0) + 1;
+    versionMap.set(relativePath, nextVersion);
+
+    const archiveDir = path.join(OUTPUT_ARCHIVE_DIR, taskId);
+    await fs.mkdir(archiveDir, { recursive: true });
+
+    const archivedFilename = buildArchivedFilename(relativePath, nextVersion);
+    const archivedPath = path.join(archiveDir, archivedFilename);
+    await fs.copyFile(sourcePath, archivedPath);
+
+    const documentId = crypto.randomUUID();
+    const now = new Date();
+
+    await db.insert(documents).values({
+      id: documentId,
+      taskId,
+      projectId: task.projectId ?? null,
+      filename: archivedFilename,
+      originalName: relativePath,
+      mimeType,
+      size: stats.size,
+      storagePath: archivedPath,
+      version: nextVersion,
+      direction: "output",
+      status: "uploaded",
+      createdAt: now,
+      updatedAt: now,
+    });
+
+    await processDocument(documentId);
+    registeredDocumentIds.push(documentId);
+  }
+
+  return registeredDocumentIds;
+}

package/src/lib/documents/processor.ts

@@ -0,0 +1,120 @@
+/**
+ * Document processing pipeline orchestrator.
+ * Registers all processors and provides the fire-and-forget processDocument entry point.
+ */
+
+import { db } from "@/lib/db";
+import { documents } from "@/lib/db/schema";
+import { eq } from "drizzle-orm";
+import { registerProcessor, getProcessor } from "./registry";
+import { processText } from "./processors/text";
+import { processPdf } from "./processors/pdf";
+import { processImage } from "./processors/image";
+import { processDocx, processPptx } from "./processors/office";
+import { processSpreadsheet } from "./processors/spreadsheet";
+
+// Register all processors by MIME type
+// Text-based formats
+const textMimeTypes = [
+  "text/plain",
+  "text/markdown",
+  "application/json",
+  "text/javascript",
+  "text/typescript",
+  "text/x-python",
+  "text/html",
+  "text/css",
+  "text/yaml",
+  "application/x-yaml",
+];
+for (const mime of textMimeTypes) {
+  registerProcessor(mime, processText);
+}
+
+// PDF
+registerProcessor("application/pdf", processPdf);
+
+// Images
+const imageMimeTypes = ["image/png", "image/jpeg", "image/gif", "image/webp"];
+for (const mime of imageMimeTypes) {
+  registerProcessor(mime, processImage);
+}
+
+// Office documents
+registerProcessor(
+  "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+  processDocx
+);
+registerProcessor(
+  "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+  processPptx
+);
+
+// Spreadsheets
+registerProcessor(
+  "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+  processSpreadsheet
+);
+registerProcessor("text/csv", processSpreadsheet);
+
+/**
+ * Process a document asynchronously. Updates the document record with
+ * extracted text or error. Never throws — errors are captured in DB.
+ */
+export async function processDocument(documentId: string): Promise<void> {
+  const [doc] = await db
+    .select()
+    .from(documents)
+    .where(eq(documents.id, documentId));
+
+  if (!doc) return;
+
+  // Mark as processing
+  await db
+    .update(documents)
+    .set({ status: "processing", updatedAt: new Date() })
+    .where(eq(documents.id, documentId));
+
+  const processor = getProcessor(doc.mimeType);
+
+  if (!processor) {
+    // No processor for this type — mark as ready with empty text
+    await db
+      .update(documents)
+      .set({
+        status: "ready",
+        extractedText: null,
+        processingError: `No processor for MIME type: ${doc.mimeType}`,
+        updatedAt: new Date(),
+      })
+      .where(eq(documents.id, documentId));
+    return;
+  }
+
+  try {
+    const result = await processor(doc.storagePath);
+
+    await db
+      .update(documents)
+      .set({
+        status: "ready",
+        extractedText: result.extractedText,
+        processedPath: result.processedPath ?? null,
+        processingError: null,
+        updatedAt: new Date(),
+      })
+      .where(eq(documents.id, documentId));
+  } catch (error: unknown) {
+    const errorMessage =
+      error instanceof Error ? error.message : String(error);
+
+    await db
+      .update(documents)
+      .set({
+        status: "error",
+        processingError: errorMessage,
+        updatedAt: new Date(),
+      })
+      .where(eq(documents.id, documentId));
+  }
+}

package/src/lib/documents/processors/image.ts

@@ -0,0 +1,21 @@
+import { readFile } from "fs/promises";
+import type { ProcessorResult } from "../registry";
+
+const SUPPORTED_FORMATS = new Set(["png", "jpg", "jpeg", "gif", "webp"]);
+
+/** Extract image dimensions metadata — agents use the file path to view images */
+export async function processImage(filePath: string): Promise<ProcessorResult> {
+  const { imageSize } = await import("image-size");
+  const buffer = await readFile(filePath);
+  const dimensions = imageSize(new Uint8Array(buffer));
+
+  if (dimensions.type && !SUPPORTED_FORMATS.has(dimensions.type)) {
+    throw new Error(`Unsupported image format: ${dimensions.type}`);
+  }
+
+  const meta = [
+    `Image: ${dimensions.width}x${dimensions.height}`,
+    `Format: ${dimensions.type}`,
+  ].join("\n");
+  return { extractedText: meta };
+}

package/src/lib/documents/processors/office.ts

@@ -0,0 +1,36 @@
+import { readFile } from "fs/promises";
+import type { ProcessorResult } from "../registry";
+
+/** Extract text from DOCX files using mammoth */
+export async function processDocx(filePath: string): Promise<ProcessorResult> {
+  const mammoth = await import("mammoth");
+  const buffer = await readFile(filePath);
+  const result = await mammoth.extractRawText({ buffer });
+  return { extractedText: result.value };
+}
+
+/** Extract text from PPTX files by parsing the XML slide contents */
+export async function processPptx(filePath: string): Promise<ProcessorResult> {
+  const JSZip = (await import("jszip")).default;
+  const buffer = await readFile(filePath);
+  const zip = await JSZip.loadAsync(buffer);
+
+  const slideTexts: string[] = [];
+  const slideFiles = Object.keys(zip.files)
+    .filter((name) => name.match(/^ppt\/slides\/slide\d+\.xml$/))
+    .sort();
+
+  for (const slideFile of slideFiles) {
+    const xml = await zip.files[slideFile].async("text");
+    // Extract text between <a:t> tags
+    const texts = xml.match(/<a:t>([^<]*)<\/a:t>/g);
+    if (texts) {
+      const slideText = texts
+        .map((t) => t.replace(/<\/?a:t>/g, ""))
+        .join(" ");
+      slideTexts.push(slideText);
+    }
+  }
+
+  return { extractedText: slideTexts.join("\n\n") };
+}