speqs 0.6.0 → 0.7.0

package/dist/lib/local-sim/types.d.ts

@@ -0,0 +1,232 @@
+/**
+ * Types for local browser simulation protocol.
+ *
+ * Aligned with backend contract (app/api/simulation/local/models.py).
+ * CLI captures observations locally, sends to Speqs API for LLM reasoning,
+ * receives actions with node_ids, resolves elements locally via CDP.
+ */
+export interface LocalSimInitRequest {
+    tester_id: string;
+    study_id: string;
+    product_id: string;
+    iteration_id: string;
+}
+export interface IterationDetails {
+    url: string;
+    platform: string;
+    screen_format: "desktop" | "mobile_portrait";
+    locale?: string;
+}
+export interface LocalSimInitResponse {
+    tester_id: string;
+    study_id: string;
+    product_id: string;
+    assignments: LocalSimAssignment[];
+    tester_background: Record<string, unknown> | null;
+    tester_language: string | null;
+    config: Record<string, unknown>;
+    context_values: ContextValue[];
+    max_interactions: number;
+    agent_model: string;
+    dom_model: string;
+    llm_provider: string;
+    iteration_details: IterationDetails | null;
+}
+export interface LocalSimAssignment {
+    id: string;
+    name: string;
+    instructions: string;
+    sequence: number;
+}
+export interface ContextValue {
+    name: string;
+    type: "var" | "secret";
+    value: string | null;
+    description?: string;
+}
+export interface HistoryEntry {
+    comment: string;
+    action_description: string;
+    location: string;
+    sentiment: string;
+    interacted_with: string | null;
+    is_user_instruction?: boolean;
+}
+export interface ForwardEntry {
+    type: string;
+    content: string;
+}
+export interface LocalSimStepRequest {
+    tester_id: string;
+    product_id: string;
+    assignment_name: string;
+    assignment_instructions: string;
+    screenshot: string;
+    accessibility_tree: string;
+    current_url: string;
+    screen_width: number;
+    screen_height: number;
+    interaction_count: number;
+    history: HistoryEntry[];
+    forwards: ForwardEntry[];
+    tester_background: Record<string, unknown> | null;
+    tester_language: string | null;
+    context_values: ContextValue[];
+    agent_model: string | null;
+    dom_model: string | null;
+    llm_provider: string | null;
+    user_instruction?: string | null;
+}
+export interface LocalStepAction {
+    type: string;
+    element_name: string | null;
+    element_description: string | null;
+    element_type: string | null;
+    node_id: string | null;
+    node_description: string | null;
+    value: string | null;
+    value_type: string | null;
+    mode: string | null;
+    submit: boolean | null;
+    direction: string | null;
+    amount: string | null;
+    count: number | null;
+    duration_ms: number | null;
+    thoughts: string | null;
+}
+/** Raw backend step response — output is nested, actions are separate. */
+export interface LocalSimStepResponseRaw {
+    output: {
+        comment: string;
+        sentiment: string;
+        sentiment_valence?: number;
+        sentiment_intensity?: number;
+        current_location: string;
+        effort_seconds: number;
+        assignment_completed: boolean;
+        action: {
+            actions: Array<{
+                type: string;
+                element?: {
+                    name: string;
+                    type?: string;
+                    description?: string;
+                };
+                value?: string;
+                value_type?: string;
+                mode?: string;
+                submit?: boolean;
+                direction?: string;
+                amount?: string;
+                count?: number;
+                duration_ms?: number;
+                thoughts?: string;
+            }>;
+        };
+    };
+    resolved_actions: LocalStepAction[];
+    loop_detected: boolean;
+}
+/** Normalized step response used by the loop. */
+export interface LocalSimStepResponse {
+    comment: string;
+    sentiment: string;
+    sentiment_valence: number;
+    sentiment_intensity: number;
+    current_location: string;
+    effort_seconds: number;
+    assignment_completed: boolean;
+    actions: LocalStepAction[];
+    loop_detected: boolean;
+}
+export interface SentimentData {
+    label: string;
+    valence: number;
+    intensity: number;
+}
+export interface ActionData {
+    action_type: string;
+    element_label: string | null;
+    element_type: string | null;
+    coordinates: {
+        x: number;
+        y: number;
+    } | null;
+    data: Record<string, unknown> | null;
+    order: number;
+}
+export interface RecordInteraction {
+    step: number;
+    assignment_id: string;
+    screenshot_base64?: string;
+    screenshot_url?: string;
+    frame_version_id?: string;
+    timestamp_ms: number;
+    comment: string | null;
+    url: string | null;
+    sentiment: SentimentData;
+    actions: ActionData[];
+    current_location: string | null;
+    assignment_completed: boolean;
+}
+export interface AssignmentStatusUpdate {
+    assignment_id: string;
+    status: string;
+    step_count: number;
+}
+export interface LocalSimRecordRequest {
+    tester_id: string;
+    product_id: string;
+    interactions: RecordInteraction[];
+    final_status: string;
+    assignment_statuses: AssignmentStatusUpdate[];
+}
+export interface LocalSimRecordResponse {
+    tester_id: string;
+    interactions_created: number;
+    credits_consumed: number;
+}
+export interface ActionResult {
+    success: boolean;
+    elementName: string | null;
+    coordinates: {
+        x: number;
+        y: number;
+    } | null;
+}
+export interface LocalSimBrowserOptions {
+    headed: boolean;
+    slowMo?: number;
+    devtools?: boolean;
+    viewport: {
+        width: number;
+        height: number;
+    };
+    locale?: string;
+    screenFormat: "desktop" | "mobile_portrait";
+}
+/** Raw accessibility tree data stored for CDP node resolution. */
+export interface TreeData {
+    simplified: string;
+    nodeMap: Map<string, {
+        backendNodeId: number;
+        frameIndex: number;
+    }>;
+}
+/**
+ * Session state cached from init, passed per step.
+ * Avoids DB lookups on the backend hot path.
+ */
+export interface SessionState {
+    tester_id: string;
+    study_id: string;
+    product_id: string;
+    assignments: LocalSimAssignment[];
+    tester_background: Record<string, unknown> | null;
+    tester_language: string | null;
+    context_values: ContextValue[];
+    max_interactions: number;
+    agent_model: string;
+    dom_model: string;
+    llm_provider: string;
+}

package/dist/lib/local-sim/types.js

@@ -0,0 +1,8 @@
+/**
+ * Types for local browser simulation protocol.
+ *
+ * Aligned with backend contract (app/api/simulation/local/models.py).
+ * CLI captures observations locally, sends to Speqs API for LLM reasoning,
+ * receives actions with node_ids, resolves elements locally via CDP.
+ */
+export {};

package/dist/lib/local-sim/upload.d.ts

@@ -0,0 +1,6 @@
+import type { ApiClient } from "../api-client.js";
+export interface ScreenshotUploadResult {
+    screenshotUrl: string;
+    screenshotId: string;
+}
+export declare function uploadScreenshot(client: ApiClient, productId: string, jpegBuffer: Buffer): Promise<ScreenshotUploadResult>;

package/dist/lib/local-sim/upload.js

@@ -0,0 +1,24 @@
+import { randomUUID } from "node:crypto";
+export async function uploadScreenshot(client, productId, jpegBuffer) {
+    const screenshotId = randomUUID();
+    // Step 1: Get signed URL from backend
+    const resp = await client.localSimScreenshotUpload({
+        product_id: productId,
+        screenshot_id: screenshotId,
+        content_type: "image/jpeg",
+    });
+    // Step 2: PUT raw JPEG bytes directly to Supabase Storage
+    const putResp = await fetch(resp.upload_info.signed_upload_url, {
+        method: "PUT",
+        headers: {
+            "Content-Type": "image/jpeg",
+            "Content-Length": String(jpegBuffer.byteLength),
+        },
+        body: jpegBuffer,
+        signal: AbortSignal.timeout(30_000),
+    });
+    if (!putResp.ok) {
+        throw new Error(`Screenshot upload failed (HTTP ${putResp.status})`);
+    }
+    return { screenshotUrl: resp.screenshot_url, screenshotId };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "speqs",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "The command-line interface for Speqs",
   "type": "module",
   "bin": {
@@ -33,7 +33,8 @@
     "email": "support@speqs.io"
   },
   "dependencies": {
-    "commander": "^13.0.0"
+    "commander": "^13.0.0",
+    "playwright-core": "^1.58.2"
   },
   "devDependencies": {
     "@types/node": "^22.0.0",