stagent 0.6.2 → 0.7.0

package/src/lib/tables/history.ts

@@ -0,0 +1,115 @@
+/**
+ * Row version history — snapshots previous state before mutations.
+ */
+
+import { randomUUID } from "crypto";
+import { db } from "@/lib/db";
+import { userTableRowHistory, userTableRows } from "@/lib/db/schema";
+import { eq, desc } from "drizzle-orm";
+
+/**
+ * Snapshot a row's current data before updating it.
+ * Called from row mutation code paths.
+ */
+export function snapshotBeforeUpdate(
+  rowId: string,
+  tableId: string,
+  previousData: string,
+  changedBy: string = "user"
+): void {
+  db.insert(userTableRowHistory)
+    .values({
+      id: randomUUID(),
+      rowId,
+      tableId,
+      previousData,
+      changedBy,
+      changeType: "update",
+      createdAt: new Date(),
+    })
+    .run();
+}
+
+/**
+ * Snapshot a row's data before deleting it.
+ */
+export function snapshotBeforeDelete(
+  rowId: string,
+  tableId: string,
+  previousData: string,
+  changedBy: string = "user"
+): void {
+  db.insert(userTableRowHistory)
+    .values({
+      id: randomUUID(),
+      rowId,
+      tableId,
+      previousData,
+      changedBy,
+      changeType: "delete",
+      createdAt: new Date(),
+    })
+    .run();
+}
+
+/**
+ * Get version history for a specific row.
+ */
+export function getRowHistory(rowId: string, limit: number = 50) {
+  return db
+    .select()
+    .from(userTableRowHistory)
+    .where(eq(userTableRowHistory.rowId, rowId))
+    .orderBy(desc(userTableRowHistory.createdAt))
+    .limit(limit)
+    .all();
+}
+
+/**
+ * Get recent history for an entire table.
+ */
+export function getTableHistory(tableId: string, limit: number = 100) {
+  return db
+    .select()
+    .from(userTableRowHistory)
+    .where(eq(userTableRowHistory.tableId, tableId))
+    .orderBy(desc(userTableRowHistory.createdAt))
+    .limit(limit)
+    .all();
+}
+
+/**
+ * Rollback a row to a previous version.
+ * Restores the previousData from a history entry.
+ */
+export function rollbackRow(historyEntryId: string): boolean {
+  const entry = db
+    .select()
+    .from(userTableRowHistory)
+    .where(eq(userTableRowHistory.id, historyEntryId))
+    .get();
+
+  if (!entry) return false;
+
+  // Snapshot current state before rollback
+  const currentRow = db
+    .select()
+    .from(userTableRows)
+    .where(eq(userTableRows.id, entry.rowId))
+    .get();
+
+  if (currentRow) {
+    snapshotBeforeUpdate(entry.rowId, entry.tableId, currentRow.data, "rollback");
+  }
+
+  // Restore the previous data
+  db.update(userTableRows)
+    .set({
+      data: entry.previousData,
+      updatedAt: new Date(),
+    })
+    .where(eq(userTableRows.id, entry.rowId))
+    .run();
+
+  return true;
+}

package/src/lib/tables/import.ts

@@ -0,0 +1,343 @@
+/**
+ * Table import module — extract structured data from documents,
+ * infer column types, and batch-import rows.
+ */
+
+import { readFile } from "fs/promises";
+import { randomUUID } from "crypto";
+import { db } from "@/lib/db";
+import {
+  documents,
+  userTableImports,
+  tableDocumentInputs,
+} from "@/lib/db/schema";
+import { eq } from "drizzle-orm";
+import { addRows } from "@/lib/data/tables";
+import type { ColumnDataType } from "@/lib/constants/table-status";
+
+// ── Type inference patterns ──────────────────────────────────────────
+
+const EMAIL_RE = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/;
+const URL_RE = /^https?:\/\/.+/;
+const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}/;
+const US_DATE_RE = /^\d{1,2}\/\d{1,2}\/\d{2,4}$/;
+const EU_DATE_RE = /^\d{1,2}\.\d{1,2}\.\d{2,4}$/;
+const BOOLEAN_VALUES = new Set(["true", "false", "yes", "no", "1", "0"]);
+const CURRENCY_RE = /^[$€£¥]?\s?[\d,]+\.?\d*$/;
+const PERCENT_RE = /^[\d.]+%$/;
+
+interface ExtractedData {
+  headers: string[];
+  rows: Record<string, string>[];
+}
+
+interface InferredColumn {
+  name: string;
+  displayName: string;
+  dataType: ColumnDataType;
+  config?: Record<string, unknown>;
+}
+
+interface ImportResult {
+  importId: string;
+  rowsImported: number;
+  rowsSkipped: number;
+  errors: Array<{ row: number; error: string }>;
+}
+
+// ── Extract structured data from a document ──────────────────────────
+
+export async function extractStructuredData(
+  documentId: string
+): Promise<ExtractedData> {
+  const doc = db
+    .select()
+    .from(documents)
+    .where(eq(documents.id, documentId))
+    .get();
+
+  if (!doc) throw new Error("Document not found");
+
+  const mime = doc.mimeType;
+  const buffer = await readFile(doc.storagePath);
+
+  if (
+    mime === "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" ||
+    mime === "application/vnd.ms-excel"
+  ) {
+    return extractFromExcel(buffer);
+  }
+
+  if (mime === "text/csv" || mime === "text/tab-separated-values") {
+    const delimiter = mime === "text/tab-separated-values" ? "\t" : ",";
+    return extractFromCsv(buffer.toString("utf-8"), delimiter);
+  }
+
+  throw new Error(`Unsupported mime type for import: ${mime}`);
+}
+
+async function extractFromExcel(buffer: Buffer): Promise<ExtractedData> {
+  const ExcelJS = await import("exceljs");
+  const workbook = new ExcelJS.Workbook();
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  await workbook.xlsx.load(buffer as any);
+
+  const worksheet = workbook.worksheets[0];
+  if (!worksheet) throw new Error("No worksheets found");
+
+  const headers: string[] = [];
+  const rows: Record<string, string>[] = [];
+
+  worksheet.eachRow((row, rowNumber) => {
+    const values = (row.values as (string | number | null | undefined)[]).slice(1);
+
+    if (rowNumber === 1) {
+      // First row = headers
+      for (const v of values) {
+        headers.push(String(v ?? `col_${headers.length}`).trim());
+      }
+    } else {
+      const rowData: Record<string, string> = {};
+      values.forEach((v, i) => {
+        if (i < headers.length) {
+          rowData[headers[i]] = v == null ? "" : String(v);
+        }
+      });
+      rows.push(rowData);
+    }
+  });
+
+  return { headers, rows };
+}
+
+function extractFromCsv(
+  content: string,
+  delimiter: string
+): ExtractedData {
+  const lines = content.split(/\r?\n/).filter((l) => l.trim());
+  if (lines.length === 0) throw new Error("Empty file");
+
+  // Simple CSV parser (handles basic cases, not quoted fields with newlines)
+  function parseLine(line: string): string[] {
+    const result: string[] = [];
+    let current = "";
+    let inQuotes = false;
+
+    for (let i = 0; i < line.length; i++) {
+      const ch = line[i];
+      if (ch === '"' && !inQuotes) {
+        inQuotes = true;
+      } else if (ch === '"' && inQuotes) {
+        if (i + 1 < line.length && line[i + 1] === '"') {
+          current += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else if (ch === delimiter && !inQuotes) {
+        result.push(current.trim());
+        current = "";
+      } else {
+        current += ch;
+      }
+    }
+    result.push(current.trim());
+    return result;
+  }
+
+  const headers = parseLine(lines[0]);
+  const rows: Record<string, string>[] = [];
+
+  for (let i = 1; i < lines.length; i++) {
+    const values = parseLine(lines[i]);
+    const rowData: Record<string, string> = {};
+    headers.forEach((h, j) => {
+      rowData[h] = values[j] ?? "";
+    });
+    rows.push(rowData);
+  }
+
+  return { headers, rows };
+}
+
+// ── Infer column types from sample data ──────────────────────────────
+
+export function inferColumnTypes(
+  headers: string[],
+  sampleRows: Record<string, string>[]
+): InferredColumn[] {
+  return headers.map((header) => {
+    const values = sampleRows
+      .map((r) => r[header])
+      .filter((v) => v != null && v !== "");
+
+    if (values.length === 0) {
+      return makeColumn(header, "text");
+    }
+
+    // Check patterns across all non-empty values
+    const allEmail = values.every((v) => EMAIL_RE.test(v));
+    if (allEmail) return makeColumn(header, "email");
+
+    const allUrl = values.every((v) => URL_RE.test(v));
+    if (allUrl) return makeColumn(header, "url");
+
+    const allBoolean = values.every((v) => BOOLEAN_VALUES.has(v.toLowerCase()));
+    if (allBoolean) return makeColumn(header, "boolean");
+
+    const allDate = values.every(
+      (v) => ISO_DATE_RE.test(v) || US_DATE_RE.test(v) || EU_DATE_RE.test(v)
+    );
+    if (allDate) return makeColumn(header, "date");
+
+    const allNumber = values.every(
+      (v) => CURRENCY_RE.test(v.replace(/,/g, "")) || PERCENT_RE.test(v) || !isNaN(Number(v))
+    );
+    if (allNumber) return makeColumn(header, "number");
+
+    // Check for select: small set of repeated values
+    const unique = new Set(values);
+    if (unique.size <= 10 && unique.size < values.length * 0.5) {
+      return makeColumn(header, "select", {
+        options: Array.from(unique).sort(),
+      });
+    }
+
+    return makeColumn(header, "text");
+  });
+}
+
+function makeColumn(
+  header: string,
+  dataType: ColumnDataType,
+  config?: Record<string, unknown>
+): InferredColumn {
+  const name = header
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "_")
+    .replace(/^_|_$/g, "")
+    || `col_${Math.random().toString(36).slice(2, 6)}`;
+
+  return {
+    name,
+    displayName: header,
+    dataType,
+    config,
+  };
+}
+
+// ── Coerce values to target types ────────────────────────────────────
+
+function coerceValue(value: string, dataType: ColumnDataType): unknown {
+  if (value === "") return null;
+
+  switch (dataType) {
+    case "number": {
+      const cleaned = value.replace(/[$€£¥,%\s]/g, "").replace(/,/g, "");
+      const num = Number(cleaned);
+      return isNaN(num) ? null : num;
+    }
+    case "boolean": {
+      const lower = value.toLowerCase();
+      return lower === "true" || lower === "yes" || lower === "1";
+    }
+    case "date": {
+      // Try to normalize to ISO format
+      if (ISO_DATE_RE.test(value)) return value;
+      // US format: MM/DD/YYYY
+      const usMatch = value.match(/^(\d{1,2})\/(\d{1,2})\/(\d{2,4})$/);
+      if (usMatch) {
+        const year = usMatch[3].length === 2 ? `20${usMatch[3]}` : usMatch[3];
+        return `${year}-${usMatch[1].padStart(2, "0")}-${usMatch[2].padStart(2, "0")}`;
+      }
+      // EU format: DD.MM.YYYY
+      const euMatch = value.match(/^(\d{1,2})\.(\d{1,2})\.(\d{2,4})$/);
+      if (euMatch) {
+        const year = euMatch[3].length === 2 ? `20${euMatch[3]}` : euMatch[3];
+        return `${year}-${euMatch[2].padStart(2, "0")}-${euMatch[1].padStart(2, "0")}`;
+      }
+      return value;
+    }
+    default:
+      return value;
+  }
+}
+
+// ── Batch import rows ────────────────────────────────────────────────
+
+export async function importRows(
+  tableId: string,
+  rawRows: Record<string, string>[],
+  columnMapping: InferredColumn[]
+): Promise<ImportResult> {
+  const importId = randomUUID();
+  const errors: Array<{ row: number; error: string }> = [];
+  let rowsImported = 0;
+
+  // Process in chunks of 100
+  const CHUNK_SIZE = 100;
+  for (let i = 0; i < rawRows.length; i += CHUNK_SIZE) {
+    const chunk = rawRows.slice(i, i + CHUNK_SIZE);
+    const validRows: Array<{ data: Record<string, unknown> }> = [];
+
+    for (let j = 0; j < chunk.length; j++) {
+      const rawRow = chunk[j];
+      const rowNum = i + j + 2; // +2 for 1-indexed + header row
+
+      try {
+        const data: Record<string, unknown> = {};
+        for (const col of columnMapping) {
+          const rawValue = rawRow[col.displayName] ?? "";
+          data[col.name] = coerceValue(rawValue, col.dataType);
+        }
+        validRows.push({ data });
+      } catch (err) {
+        errors.push({
+          row: rowNum,
+          error: err instanceof Error ? err.message : "Unknown error",
+        });
+      }
+    }
+
+    if (validRows.length > 0) {
+      await addRows(tableId, validRows);
+      rowsImported += validRows.length;
+    }
+  }
+
+  return { importId, rowsImported, rowsSkipped: errors.length, errors };
+}
+
+// ── Create import audit record ───────────────────────────────────────
+
+export async function createImportRecord(
+  tableId: string,
+  documentId: string,
+  result: ImportResult
+): Promise<void> {
+  const now = new Date();
+
+  // Create audit record
+  await db.insert(userTableImports).values({
+    id: result.importId,
+    tableId,
+    documentId,
+    rowCount: result.rowsImported,
+    errorCount: result.rowsSkipped,
+    errors: result.errors.length > 0 ? JSON.stringify(result.errors) : null,
+    status: result.rowsSkipped > 0 && result.rowsImported === 0 ? "failed" : "completed",
+    createdAt: now,
+  });
+
+  // Link document to table
+  try {
+    await db.insert(tableDocumentInputs).values({
+      id: randomUUID(),
+      tableId,
+      documentId,
+      createdAt: now,
+    });
+  } catch {
+    // Unique constraint — already linked
+  }
+}

package/src/lib/tables/query-builder.ts

@@ -0,0 +1,152 @@
+/**
+ * Query builder for user-defined table rows.
+ *
+ * Translates structured FilterSpec/SortSpec objects into SQL fragments
+ * using json_extract() to query the JSON `data` column.
+ *
+ * SECURITY: Column names are validated against the table's known column
+ * schema before being interpolated into json_extract paths. Never pass
+ * raw user input as column names without validation.
+ */
+
+import { sql, type SQL } from "drizzle-orm";
+import type { FilterSpec, SortSpec, ColumnDef } from "./types";
+
+/** Characters allowed in column names — reject anything else */
+const SAFE_COLUMN_NAME = /^[a-zA-Z_][a-zA-Z0-9_]{0,63}$/;
+
+/**
+ * Validate that a column name exists in the table's schema and is safe
+ * for interpolation into json_extract paths.
+ */
+function validateColumnName(
+  columnName: string,
+  validColumns: Set<string>
+): void {
+  if (!SAFE_COLUMN_NAME.test(columnName)) {
+    throw new Error(`Invalid column name: ${columnName}`);
+  }
+  if (!validColumns.has(columnName)) {
+    throw new Error(`Unknown column: ${columnName}`);
+  }
+}
+
+/**
+ * Build a json_extract SQL fragment for a column.
+ * Output: json_extract(data, '$.columnName')
+ */
+function jsonCol(columnName: string): SQL {
+  // Column name is already validated — safe to interpolate into the path string
+  return sql.raw(`json_extract(data, '$.${columnName}')`);
+}
+
+/**
+ * Build a WHERE clause fragment from a single filter spec.
+ */
+function buildFilterClause(
+  filter: FilterSpec,
+  validColumns: Set<string>
+): SQL {
+  validateColumnName(filter.column, validColumns);
+  const col = jsonCol(filter.column);
+
+  switch (filter.operator) {
+    case "eq":
+      return sql`${col} = ${filter.value}`;
+    case "neq":
+      return sql`${col} != ${filter.value}`;
+    case "gt":
+      return sql`${col} > ${filter.value}`;
+    case "gte":
+      return sql`${col} >= ${filter.value}`;
+    case "lt":
+      return sql`${col} < ${filter.value}`;
+    case "lte":
+      return sql`${col} <= ${filter.value}`;
+    case "contains":
+      return sql`${col} LIKE ${"%" + String(filter.value) + "%"}`;
+    case "starts_with":
+      return sql`${col} LIKE ${String(filter.value) + "%"}`;
+    case "in": {
+      const values = Array.isArray(filter.value) ? filter.value : [String(filter.value)];
+      if (values.length === 0) {
+        return sql`0 = 1`; // empty IN → always false
+      }
+      // Build parameterized IN clause
+      const placeholders = values.map((v) => sql`${v}`);
+      return sql`${col} IN (${sql.join(placeholders, sql`, `)})`;
+    }
+    case "is_empty":
+      return sql`(${col} IS NULL OR ${col} = '')`;
+    case "is_not_empty":
+      return sql`(${col} IS NOT NULL AND ${col} != '')`;
+    default:
+      throw new Error(`Unknown filter operator: ${filter.operator}`);
+  }
+}
+
+/**
+ * Build a complete WHERE clause from multiple filters (AND-joined).
+ */
+export function buildWhereClause(
+  filters: FilterSpec[],
+  validColumns: Set<string>
+): SQL | undefined {
+  if (filters.length === 0) return undefined;
+
+  const clauses = filters.map((f) => buildFilterClause(f, validColumns));
+  return sql.join(clauses, sql` AND `);
+}
+
+/**
+ * Build an ORDER BY clause from sort specs.
+ */
+export function buildOrderClause(
+  sorts: SortSpec[],
+  validColumns: Set<string>
+): SQL | undefined {
+  if (sorts.length === 0) return undefined;
+
+  const clauses = sorts.map((s) => {
+    validateColumnName(s.column, validColumns);
+    const col = jsonCol(s.column);
+    return s.direction === "desc" ? sql`${col} DESC` : sql`${col} ASC`;
+  });
+
+  return sql.join(clauses, sql`, `);
+}
+
+/**
+ * Extract the set of valid column names from a column schema.
+ */
+export function getValidColumns(columnSchema: ColumnDef[]): Set<string> {
+  return new Set(columnSchema.map((c) => c.name));
+}
+
+/**
+ * Build a full query SQL fragment for row filtering, sorting, and pagination.
+ * Returns the WHERE, ORDER BY, LIMIT, and OFFSET fragments.
+ */
+export function buildRowQuery(
+  columnSchema: ColumnDef[],
+  options: {
+    filters?: FilterSpec[];
+    sorts?: SortSpec[];
+    limit?: number;
+    offset?: number;
+  }
+): {
+  where: SQL | undefined;
+  orderBy: SQL | undefined;
+  limit: number;
+  offset: number;
+} {
+  const validColumns = getValidColumns(columnSchema);
+
+  return {
+    where: options.filters ? buildWhereClause(options.filters, validColumns) : undefined,
+    orderBy: options.sorts ? buildOrderClause(options.sorts, validColumns) : undefined,
+    limit: options.limit ?? 100,
+    offset: options.offset ?? 0,
+  };
+}