r1-create 1.2.0 → 1.3.1

package/README.md

@@ -109,6 +109,53 @@ r1.messaging.onMessage((response) => {
     console.log('Parsed response:', response.parsedData);
   }
 });
+
+// Push-to-talk using hardware long press + STT transcript forwarding
+const disablePTT = r1.messaging.enablePushToTalk({
+  onTranscript: (transcript) => {
+    console.log('User said:', transcript);
+  }
+});
+
+// Programmatic STT control
+r1.messaging.startSTTListening();
+// ...later
+r1.messaging.stopSTTListening();
+
+// Later, if needed:
+// disablePTT();
+
+// Wait for a matching response with timeout
+const llmResponse = await r1.messaging.askLLMWithTimeout(
+  'Summarize this in one sentence',
+  { wantsR1Response: false },
+  { timeoutMs: 15000 }
+);
+console.log('LLM response:', llmResponse.message);
+
+// Advanced SERP search options
+await r1.messaging.searchWeb('weather in Tokyo', {
+  tag: 'weather',
+  useLocation: true
+});
+
+// Ask R1 to email content to the user
+await r1.messaging.emailUser('Your itinerary for today...');
+
+// Analyze an image with LLM
+await r1.llm.analyzeImageBase64('Describe this image', myBase64Image);
+
+// Ask about an image (image + prompt)
+await r1.llm.askImage(myBase64Image, 'What do you see in this image?');
+
+// Transform an image (sends: "take a photo and make it <prompt>")
+await r1.llm.transformPhoto(myBase64Image, 'a watercolor painting');
+
+// Send image with no prompt
+await r1.llm.imageToAI(myBase64Image, '');
+
+// Runtime bridge checks (useful during development)
+console.log(r1.messaging.getRuntimeCapabilities());
 ```
 
 ### Media APIs

package/dist/index.d.ts

@@ -16,6 +16,7 @@ export type { HardwareEventType } from './types';
 export { AccelerometerAPI, TouchAPI, HardwareEvents, accelerometer, touch, hardwareEvents, DeviceControls, deviceControls, type DeviceControlsOptions, type ScrollWheelData, type SideButtonHandler, type ScrollWheelHandler } from './hardware';
 export { Base64Utils, R1Storage, storage } from './storage';
 export { R1Messaging, LLMHelpers, messaging, llmHelpers } from './llm';
+export type { PushToTalkOptions, SearchWebOptions, SerpSearchTag, WaitForMessageOptions, EmailOptions, ImageMessageOptions } from './llm';
 export { CSSUtils, DOMUtils, LayoutUtils, PerformanceUtils, R1Component, R1_DIMENSIONS, R1UI, ui, type UIDimensions, type UIFontSizes, type UISpacing, type UIButtonSizes, type UIColorPalette, type UIContainerOptions, type UIButtonOptions, type UITextOptions, type UIGridOptions } from './ui';
 export { CameraAPI, MicrophoneAPI, SpeakerAPI, MediaUtils, camera, microphone, speaker } from './media';
 import { R1Storage } from './storage';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,cAAc,SAAS,CAAC;AACxB,YAAY,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAGjD,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,cAAc,EACd,aAAa,EACb,KAAK,EACL,cAAc,EAEd,cAAc,EACd,cAAc,EACd,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACxB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,SAAS,EACT,OAAO,EACR,MAAM,WAAW,CAAC;AAGnB,OAAO,EACL,WAAW,EACX,UAAU,EACV,SAAS,EACT,UAAU,EACX,MAAM,OAAO,CAAC;AAGf,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,aAAa,EAEb,IAAI,EACJ,EAAE,EACF,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,aAAa,EACnB,MAAM,MAAM,CAAC;AAGd,OAAO,EACL,SAAS,EACT,aAAa,EACb,UAAU,EACV,UAAU,EACV,MAAM,EACN,UAAU,EACV,OAAO,EACR,MAAM,SAAS,CAAC;AAUjB,OAAO,EAAW,SAAS,EAAE,MAAM,WAAW,CAAC;AAK/C;;;GAGG;AACH,qBAAa,KAAK;IAEhB,SAAgB,aAAa,wCAAiB;IAC9C,SAAgB,KAAK,gCAAS;IAC9B,SAAgB,QAAQ,sCAAkB;IAC1C,SAAgB,cAAc,sCAAkB;IAGhD,SAAgB,OAAO,YAAW;IAGlC,SAAgB,SAAS,8BAAa;IACtC,SAAgB,GAAG,6BAAc;IAGjC,SAAgB,MAAM,8BAAU;IAChC,SAAgB,UAAU,kCAAc;IACxC,SAAgB,OAAO,+BAAW;IAGlC,SAAgB,EAAE,sBAAM;IAGxB,SAAgB,UAAU,iCAAiB;IAE3C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,CAAC;QACjB,aAAa,EAAE,OAAO,CAAC;QACvB,aAAa,EAAE,OAAO,CAAC;QACvB,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IAYF;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;QACvB,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC3C,GAAG,QAAQ;CAGb;AAED;;GAEG;AACH,qBAAa,QAAQ;IAGP,OAAO,CAAC,MAAM;IAF1B,OAAO,CAAC,OAAO,CAAS;gBAEJ,MAAM,EAAE;QAC1B,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;QACvB,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC3C;IAID,KAAK,IAAI,IAAI;IAOb,OAAO,IAAI,IAAI;IAOf,OAAO,CAAC,mBAAmB;CAuB5B;AAGD,eAAO,MAAM,EAAE,OAAc,CAAC;AAG9B,eAAO,MAAM,YAAY,qBAAwB,CAAC;AAElD;;;GAGG;AACH,eAAO,MAAM,WAAW,GAAI,OAAO,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,SAKtD,CAAC;AAGF,eAAe,EAAE,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,cAAc,SAAS,CAAC;AACxB,YAAY,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAGjD,OAAO,EACL,gBAAgB,EAChB,QAAQ,EACR,cAAc,EACd,aAAa,EACb,KAAK,EACL,cAAc,EAEd,cAAc,EACd,cAAc,EACd,KAAK,qBAAqB,EAC1B,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACxB,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,WAAW,EACX,SAAS,EACT,OAAO,EACR,MAAM,WAAW,CAAC;AAGnB,OAAO,EACL,WAAW,EACX,UAAU,EACV,SAAS,EACT,UAAU,EACX,MAAM,OAAO,CAAC;AACf,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,qBAAqB,EACrB,YAAY,EACZ,mBAAmB,EACpB,MAAM,OAAO,CAAC;AAGf,OAAO,EACL,QAAQ,EACR,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,aAAa,EAEb,IAAI,EACJ,EAAE,EACF,KAAK,YAAY,EACjB,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,aAAa,EACnB,MAAM,MAAM,CAAC;AAGd,OAAO,EACL,SAAS,EACT,aAAa,EACb,UAAU,EACV,UAAU,EACV,MAAM,EACN,UAAU,EACV,OAAO,EACR,MAAM,SAAS,CAAC;AAUjB,OAAO,EAAW,SAAS,EAAE,MAAM,WAAW,CAAC;AAK/C;;;GAGG;AACH,qBAAa,KAAK;IAEhB,SAAgB,aAAa,wCAAiB;IAC9C,SAAgB,KAAK,gCAAS;IAC9B,SAAgB,QAAQ,sCAAkB;IAC1C,SAAgB,cAAc,sCAAkB;IAGhD,SAAgB,OAAO,YAAW;IAGlC,SAAgB,SAAS,8BAAa;IACtC,SAAgB,GAAG,6BAAc;IAGjC,SAAgB,MAAM,8BAAU;IAChC,SAAgB,UAAU,kCAAc;IACxC,SAAgB,OAAO,+BAAW;IAGlC,SAAgB,EAAE,sBAAM;IAGxB,SAAgB,UAAU,iCAAiB;IAE3C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAejC;;OAEG;IACG,oBAAoB,IAAI,OAAO,CAAC;QACpC,OAAO,EAAE,OAAO,CAAC;QACjB,aAAa,EAAE,OAAO,CAAC;QACvB,aAAa,EAAE,OAAO,CAAC;QACvB,MAAM,EAAE,OAAO,CAAC;QAChB,UAAU,EAAE,OAAO,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC;QACjB,SAAS,EAAE,OAAO,CAAC;KACpB,CAAC;IAYF;;;OAGG;IACH,YAAY,CAAC,MAAM,EAAE;QACnB,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;QACvB,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC3C,GAAG,QAAQ;CAGb;AAED;;GAEG;AACH,qBAAa,QAAQ;IAGP,OAAO,CAAC,MAAM;IAF1B,OAAO,CAAC,OAAO,CAAS;gBAEJ,MAAM,EAAE;QAC1B,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;QACvB,SAAS,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,KAAK,IAAI,CAAC;QAChC,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC3C;IAID,KAAK,IAAI,IAAI;IAOb,OAAO,IAAI,IAAI;IAOf,OAAO,CAAC,mBAAmB;CAuB5B;AAGD,eAAO,MAAM,EAAE,OAAc,CAAC;AAG9B,eAAO,MAAM,YAAY,qBAAwB,CAAC;AAElD;;;GAGG;AACH,eAAO,MAAM,WAAW,GAAI,OAAO,CAAC,GAAG,EAAE,KAAK,KAAK,IAAI,SAKtD,CAAC;AAGF,eAAe,EAAE,CAAC"}

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;;;;;;;;;;;;;;;;AAEH,mBAAmB;AACnB,0CAAwB;AAGxB,gBAAgB;AAChB,uCAcoB;AAblB,4GAAA,gBAAgB,OAAA;AAChB,oGAAA,QAAQ,OAAA;AACR,0GAAA,cAAc,OAAA;AACd,yGAAA,aAAa,OAAA;AACb,iGAAA,KAAK,OAAA;AACL,0GAAA,cAAc,OAAA;AACd,sBAAsB;AACtB,0GAAA,cAAc,OAAA;AACd,0GAAA,cAAc,OAAA;AAOhB,eAAe;AACf,qCAImB;AAHjB,sGAAA,WAAW,OAAA;AACX,oGAAA,SAAS,OAAA;AACT,kGAAA,OAAO,OAAA;AAGT,yBAAyB;AACzB,6BAKe;AAJb,kGAAA,WAAW,OAAA;AACX,iGAAA,UAAU,OAAA;AACV,gGAAA,SAAS,OAAA;AACT,iGAAA,UAAU,OAAA;AAGZ,eAAe;AACf,2BAmBc;AAlBZ,8FAAA,QAAQ,OAAA;AACR,8FAAA,QAAQ,OAAA;AACR,iGAAA,WAAW,OAAA;AACX,sGAAA,gBAAgB,OAAA;AAChB,iGAAA,WAAW,OAAA;AACX,mGAAA,aAAa,OAAA;AACb,uBAAuB;AACvB,0FAAA,IAAI,OAAA;AACJ,wFAAA,EAAE,OAAA;AAYJ,aAAa;AACb,iCAQiB;AAPf,kGAAA,SAAS,OAAA;AACT,sGAAA,aAAa,OAAA;AACb,mGAAA,UAAU,OAAA;AACV,mGAAA,UAAU,OAAA;AACV,+FAAA,MAAM,OAAA;AACN,mGAAA,UAAU,OAAA;AACV,gGAAA,OAAO,OAAA;AAKT,yCAKoB;AACpB,uCAA+C;AAC/C,+BAA8C;AAC9C,6BAAyC;AACzC,mCAAkE;AAElE;;;GAGG;AACH,MAAa,KAAK;IAAlB;QACE,kBAAkB;QACF,kBAAa,GAAG,wBAAa,CAAC;QAC9B,UAAK,GAAG,gBAAK,CAAC;QACd,aAAQ,GAAG,yBAAc,CAAC;QAC1B,mBAAc,GAAG,yBAAc,CAAC;QAEhD,UAAU;QACM,YAAO,GAAG,iBAAO,CAAC;QAElC,oBAAoB;QACJ,cAAS,GAAG,eAAS,CAAC;QACtB,QAAG,GAAG,gBAAU,CAAC;QAEjC,QAAQ;QACQ,WAAM,GAAG,cAAM,CAAC;QAChB,eAAU,GAAG,kBAAU,CAAC;QACxB,YAAO,GAAG,eAAO,CAAC;QAElC,KAAK;QACW,OAAE,GAAG,OAAE,CAAC;QAExB,YAAY;QACI,eAAU,GAAG,kBAAa,CAAC;IAuD7C,CAAC;IArDC;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,qCAAqC;QACrC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,yBAAyB;QACzB,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,uBAAuB,kBAAa,CAAC,KAAK,IAAI,kBAAa,CAAC,MAAM,IAAI,CAAC,CAAC;QAEpF,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QASxB,OAAO;YACL,OAAO,EAAE,mBAAS,CAAC,WAAW,EAAE;YAChC,aAAa,EAAE,mBAAS,CAAC,iBAAiB,EAAE;YAC5C,aAAa,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE;YACrD,MAAM,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE;YAC/C,OAAO,EAAE,MAAM,kBAAU,CAAC,WAAW,CAAC,SAAS,CAAC;YAChD,SAAS,EAAE,OAAO,oBAAoB,KAAK,WAAW;SACvD,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,MAKZ;QACC,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;CACF;AA9ED,sBA8EC;AAED;;GAEG;AACH,MAAa,QAAQ;IAGnB,YAAoB,MAKnB;QALmB,WAAM,GAAN,MAAM,CAKzB;QAPO,YAAO,GAAG,KAAK,CAAC;QAQtB,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;IAC5B,CAAC;IAEO,mBAAmB;QACzB,yBAAyB;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,eAAS,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7C,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB;gBAClC,WAAW;gBACX,gBAAgB;gBAChB,cAAc;gBACd,UAAU;gBACV,YAAY;aACb,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACrB,yBAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF;AAjDD,4BAiDC;AAED,yCAAyC;AAC5B,QAAA,EAAE,GAAG,IAAI,KAAK,EAAE,CAAC;AAE9B,+BAA+B;AACxB,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,UAAE,CAAC,UAAU,EAAE,CAAC;AAArC,QAAA,YAAY,gBAAyB;AAElD;;;GAGG;AACI,MAAM,WAAW,GAAG,CAAC,KAA2B,EAAE,EAAE;IACzD,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,UAAE,CAAC,UAAU,EAAE,CAAC;QACtB,KAAK,CAAC,UAAE,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AALW,QAAA,WAAW,eAKtB;AAEF,iBAAiB;AACjB,kBAAe,UAAE,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG;;;;;;;;;;;;;;;;;AAEH,mBAAmB;AACnB,0CAAwB;AAGxB,gBAAgB;AAChB,uCAcoB;AAblB,4GAAA,gBAAgB,OAAA;AAChB,oGAAA,QAAQ,OAAA;AACR,0GAAA,cAAc,OAAA;AACd,yGAAA,aAAa,OAAA;AACb,iGAAA,KAAK,OAAA;AACL,0GAAA,cAAc,OAAA;AACd,sBAAsB;AACtB,0GAAA,cAAc,OAAA;AACd,0GAAA,cAAc,OAAA;AAOhB,eAAe;AACf,qCAImB;AAHjB,sGAAA,WAAW,OAAA;AACX,oGAAA,SAAS,OAAA;AACT,kGAAA,OAAO,OAAA;AAGT,yBAAyB;AACzB,6BAKe;AAJb,kGAAA,WAAW,OAAA;AACX,iGAAA,UAAU,OAAA;AACV,gGAAA,SAAS,OAAA;AACT,iGAAA,UAAU,OAAA;AAWZ,eAAe;AACf,2BAmBc;AAlBZ,8FAAA,QAAQ,OAAA;AACR,8FAAA,QAAQ,OAAA;AACR,iGAAA,WAAW,OAAA;AACX,sGAAA,gBAAgB,OAAA;AAChB,iGAAA,WAAW,OAAA;AACX,mGAAA,aAAa,OAAA;AACb,uBAAuB;AACvB,0FAAA,IAAI,OAAA;AACJ,wFAAA,EAAE,OAAA;AAYJ,aAAa;AACb,iCAQiB;AAPf,kGAAA,SAAS,OAAA;AACT,sGAAA,aAAa,OAAA;AACb,mGAAA,UAAU,OAAA;AACV,mGAAA,UAAU,OAAA;AACV,+FAAA,MAAM,OAAA;AACN,mGAAA,UAAU,OAAA;AACV,gGAAA,OAAO,OAAA;AAKT,yCAKoB;AACpB,uCAA+C;AAC/C,+BAA8C;AAC9C,6BAAyC;AACzC,mCAAkE;AAElE;;;GAGG;AACH,MAAa,KAAK;IAAlB;QACE,kBAAkB;QACF,kBAAa,GAAG,wBAAa,CAAC;QAC9B,UAAK,GAAG,gBAAK,CAAC;QACd,aAAQ,GAAG,yBAAc,CAAC;QAC1B,mBAAc,GAAG,yBAAc,CAAC;QAEhD,UAAU;QACM,YAAO,GAAG,iBAAO,CAAC;QAElC,oBAAoB;QACJ,cAAS,GAAG,eAAS,CAAC;QACtB,QAAG,GAAG,gBAAU,CAAC;QAEjC,QAAQ;QACQ,WAAM,GAAG,cAAM,CAAC;QAChB,eAAU,GAAG,kBAAU,CAAC;QACxB,YAAO,GAAG,eAAO,CAAC;QAElC,KAAK;QACW,OAAE,GAAG,OAAE,CAAC;QAExB,YAAY;QACI,eAAU,GAAG,kBAAa,CAAC;IAuD7C,CAAC;IArDC;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,qCAAqC;QACrC,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QAED,yBAAyB;QACzB,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,uBAAuB,kBAAa,CAAC,KAAK,IAAI,kBAAa,CAAC,MAAM,IAAI,CAAC,CAAC;QAEpF,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,oBAAoB;QASxB,OAAO;YACL,OAAO,EAAE,mBAAS,CAAC,WAAW,EAAE;YAChC,aAAa,EAAE,mBAAS,CAAC,iBAAiB,EAAE;YAC5C,aAAa,EAAE,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE;YACrD,MAAM,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;YACvC,UAAU,EAAE,MAAM,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE;YAC/C,OAAO,EAAE,MAAM,kBAAU,CAAC,WAAW,CAAC,SAAS,CAAC;YAChD,SAAS,EAAE,OAAO,oBAAoB,KAAK,WAAW;SACvD,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,MAKZ;QACC,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;CACF;AA9ED,sBA8EC;AAED;;GAEG;AACH,MAAa,QAAQ;IAGnB,YAAoB,MAKnB;QALmB,WAAM,GAAN,MAAM,CAKzB;QAPO,YAAO,GAAG,KAAK,CAAC;QAQtB,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAC7B,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAEzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;IAC5B,CAAC;IAEO,mBAAmB;QACzB,yBAAyB;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1B,eAAS,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7C,CAAC;QAED,gCAAgC;QAChC,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;YAChC,MAAM,MAAM,GAAwB;gBAClC,WAAW;gBACX,gBAAgB;gBAChB,cAAc;gBACd,UAAU;gBACV,YAAY;aACb,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;gBACrB,yBAAc,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBAC5B,IAAI,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF;AAjDD,4BAiDC;AAED,yCAAyC;AAC5B,QAAA,EAAE,GAAG,IAAI,KAAK,EAAE,CAAC;AAE9B,+BAA+B;AACxB,MAAM,YAAY,GAAG,GAAG,EAAE,CAAC,UAAE,CAAC,UAAU,EAAE,CAAC;AAArC,QAAA,YAAY,gBAAyB;AAElD;;;GAGG;AACI,MAAM,WAAW,GAAG,CAAC,KAA2B,EAAE,EAAE;IACzD,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,UAAE,CAAC,UAAU,EAAE,CAAC;QACtB,KAAK,CAAC,UAAE,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AALW,QAAA,WAAW,eAKtB;AAEF,iBAAiB;AACjB,kBAAe,UAAE,CAAC"}

package/dist/llm/index.d.ts

@@ -2,7 +2,7 @@
  * LLM and messaging module for R1 AI integration
  * Provides structured messaging and LLM interaction capabilities
  */
-import type { PluginMessageResponse } from '../types';
+import type { PluginMessageResponse, SpeechSynthesisOptions } from '../types';
 export interface LLMOptions {
     wantsR1Response?: boolean;
     wantsJournalEntry?: boolean;
@@ -13,18 +13,45 @@ export interface MessageOptions extends LLMOptions {
     pluginId?: string;
     imageBase64?: string;
 }
+export type SerpSearchTag = 'search' | 'image' | 'finance' | 'jobs' | 'weather' | 'hotels';
+export interface SearchWebOptions extends Omit<MessageOptions, 'useSerpAPI'> {
+    useLocation?: boolean;
+    tag?: SerpSearchTag;
+}
+export interface WaitForMessageOptions {
+    timeoutMs?: number;
+    predicate?: (response: PluginMessageResponse & {
+        parsedData?: any;
+    }) => boolean;
+}
+export interface EmailOptions extends Omit<MessageOptions, 'useLLM'> {
+    instructionPrefix?: string;
+}
+export interface ImageMessageOptions extends Omit<MessageOptions, 'imageBase64' | 'useLLM'> {
+}
 /**
  * Type-safe LLM response handler
  */
 export type MessageHandler<T = any> = (response: PluginMessageResponse & {
     parsedData?: T;
 }) => void;
+export interface PushToTalkOptions {
+    startEvent?: string;
+    endEvent?: string;
+    processMessage?: string;
+    autoForwardTranscript?: boolean;
+    onTranscript?: (transcript: string, response: PluginMessageResponse & {
+        parsedData?: any;
+    }) => void;
+}
 /**
  * LLM and messaging API for R1 interactions
  */
 export declare class R1Messaging {
     private messageHandlers;
     private isInitialized;
+    private pushToTalkCleanup;
+    private sttListening;
     constructor();
     /**
      * Send a simple message to the server
@@ -43,7 +70,7 @@ export declare class R1Messaging {
      * @param query Search query
      * @param options Additional options
      */
-    searchWeb(query: string, options?: Omit<MessageOptions, 'useSerpAPI'>): Promise<void>;
+    searchWeb(query: string, options?: SearchWebOptions): Promise<void>;
     /**
      * Send text for text-to-speech output (without LLM processing)
      * @param text Text to speak
@@ -76,11 +103,95 @@ export declare class R1Messaging {
      * Remove all message handlers
      */
     removeAllHandlers(): void;
+    /**
+     * Wait for the next incoming plugin message.
+     * Useful for request/response flows with timeout handling.
+     */
+    waitForNextMessage(options?: WaitForMessageOptions): Promise<PluginMessageResponse & {
+        parsedData?: any;
+    }>;
+    /**
+     * Send a message to LLM and await the next matching response.
+     */
+    askLLMWithTimeout(message: string, options?: LLMOptions, waitOptions?: WaitForMessageOptions): Promise<PluginMessageResponse & {
+        parsedData?: any;
+    }>;
+    /**
+     * Ask the LLM to email content to the user.
+     */
+    emailUser(content: string, options?: EmailOptions): Promise<void>;
+    /**
+     * Send an image to AI with an optional prompt.
+     *
+     * Common patterns:
+     * - Ask about image: provide a question prompt.
+     * - Transform image: use a prompt like "take a photo and make it ...".
+     * - No-prompt processing: pass empty prompt.
+     */
+    sendImageToAI(imageBase64: string, prompt?: string, options?: ImageMessageOptions): Promise<void>;
+    /**
+     * Ask AI about an image by sending image + prompt.
+     */
+    askAboutImage(imageBase64: string, prompt: string, options?: ImageMessageOptions): Promise<void>;
+    /**
+     * Transform an image using the common R1 phrasing.
+     * Sends: "take a photo and make it <prompt>" + image.
+     */
+    transformImage(imageBase64: string, transformPrompt: string, options?: ImageMessageOptions): Promise<void>;
+    /**
+     * Runtime availability checks for messaging-related R1 bridges.
+     */
+    getRuntimeCapabilities(): {
+        pluginMessageHandler: boolean;
+        creationVoiceHandler: boolean;
+        closeWebView: boolean;
+    };
+    /**
+     * Start STT listening programmatically.
+     * Equivalent to: `CreationVoiceHandler.postMessage('start')`
+     */
+    startSTTListening(): void;
+    /**
+     * Stop STT listening programmatically.
+     * Equivalent to: `CreationVoiceHandler.postMessage('stop')`
+     */
+    stopSTTListening(): void;
+    /**
+     * Returns whether STT is currently listening based on SDK-triggered state.
+     */
+    isSTTListening(): boolean;
+    /**
+     * Enable push-to-talk using hardware long press events and STT messages.
+     *
+     * Default behavior:
+     * - `longPressStart` => `CreationVoiceHandler.postMessage('start')`
+     * - `longPressEnd` => `CreationVoiceHandler.postMessage('stop')`
+     * - On `sttEnded` with transcript, forwards:
+     *   `{ message: 'process_voice_input', transcript }` via PluginMessageHandler
+     */
+    enablePushToTalk(options?: PushToTalkOptions): () => void;
+    /**
+     * Disable push-to-talk listeners previously configured via enablePushToTalk.
+     */
+    disablePushToTalk(): void;
     /**
      * Close the current plugin/webview
      */
     closePlugin(): void;
     private initializeMessageHandler;
+    /**
+     * Normalize image payload for R1 bridge compatibility.
+     * Accepts either full data URL or raw base64 and returns a data URL string.
+     */
+    private normalizeImagePayload;
+    /**
+     * Generate audio file from text-to-speech (browser only)
+     * Uses Web Speech API to synthesize speech and capture as audio blob
+     * @param text Text to convert to audio
+     * @param options Speech synthesis options
+     * @returns Promise resolving to audio blob or null if not supported
+     */
+    textToSpeechAudio(text: string, options?: SpeechSynthesisOptions): Promise<Blob | null>;
 }
 /**
  * Convenient helper functions for common LLM interactions
@@ -98,6 +209,27 @@ export declare class LLMHelpers {
      * @param data Optional data to analyze
      */
     analyzeData(prompt: string, data?: any): Promise<void>;
+    /**
+     * Analyze a base64-encoded image with the LLM.
+     * @param prompt Analysis prompt
+     * @param imageBase64 Base64 image string (without data URL prefix)
+     * @param options Additional message options
+     */
+    analyzeImageBase64(prompt: string, imageBase64: string, options?: Omit<MessageOptions, 'imageBase64' | 'useLLM'>): Promise<void>;
+    /**
+     * Send an image to AI with optional prompt.
+     * Empty prompt triggers no-prompt image processing.
+     */
+    imageToAI(imageBase64: string, prompt?: string, options?: ImageMessageOptions): Promise<void>;
+    /**
+     * Ask AI about an image (image + prompt).
+     */
+    askImage(imageBase64: string, prompt: string, options?: ImageMessageOptions): Promise<void>;
+    /**
+     * Transform an image with phrasing:
+     * "take a photo and make it <prompt>".
+     */
+    transformPhoto(imageBase64: string, prompt: string, options?: ImageMessageOptions): Promise<void>;
     /**
      * Ask LLM to perform a task and speak the result
      * @param task Task description
@@ -115,6 +247,12 @@ export declare class LLMHelpers {
      * @param context Current UI context
      */
     getUISuggestions(context: string): Promise<void>;
+    /**
+     * Generate audio file from text-to-speech (browser only)
+     * @param text Text to convert to audio
+     * @param options Speech synthesis options
+     */
+    textToSpeechAudio(text: string, options?: SpeechSynthesisOptions): Promise<Blob | null>;
 }
 export declare const messaging: R1Messaging;
 export declare const llmHelpers: LLMHelpers;

package/dist/llm/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAiB,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAErE,MAAM,WAAW,UAAU;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,cAAe,SAAQ,UAAU;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,qBAAqB,GAAG;IAAE,UAAU,CAAC,EAAE,CAAC,CAAA;CAAE,KAAK,IAAI,CAAC;AAErG;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,aAAa,CAAS;;IAM9B;;;;OAIG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa/E;;;;OAIG;IACG,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtE;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,YAAY,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkB/F;;;;OAIG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,iBAAiB,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9G;;;;OAIG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,GAAE,OAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjF;;;;OAIG;IACG,UAAU,CAAC,CAAC,GAAG,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnF;;;OAGG;IACH,SAAS,CAAC,CAAC,GAAG,GAAG,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,IAAI;IAIpD;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI;IAIzC;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAIzB;;OAEG;IACH,WAAW,IAAI,IAAI;IAMnB,OAAO,CAAC,wBAAwB;CAiCjC;AAED;;GAEG;AACH,qBAAa,UAAU;IACT,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,WAAW;IAE1C;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAMtC;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5D;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,GAAE,OAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7E;;;;OAIG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,GAAE,OAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;OAGG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAMvD;AAGD,eAAO,MAAM,SAAS,aAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,YAA4B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/llm/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAiB,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAE7F,MAAM,WAAW,UAAU;IACzB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,cAAe,SAAQ,UAAU;IAChD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,OAAO,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE3F,MAAM,WAAW,gBAAiB,SAAQ,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC;IAC1E,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,GAAG,CAAC,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,qBAAqB,GAAG;QAAE,UAAU,CAAC,EAAE,GAAG,CAAA;KAAE,KAAK,OAAO,CAAC;CACjF;AAED,MAAM,WAAW,YAAa,SAAQ,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC;IAClE,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,mBAAoB,SAAQ,IAAI,CAAC,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAC;CAAG;AAE9F;;GAEG;AACH,MAAM,MAAM,cAAc,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,qBAAqB,GAAG;IAAE,UAAU,CAAC,EAAE,CAAC,CAAA;CAAE,KAAK,IAAI,CAAC;AAErG,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,qBAAqB,GAAG;QAAE,UAAU,CAAC,EAAE,GAAG,CAAA;KAAE,KAAK,IAAI,CAAC;CACrG;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,eAAe,CAAkC;IACzD,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,YAAY,CAAS;;IAM7B;;;;OAIG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa/E;;;;OAIG;IACG,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtE;;;;OAIG;IACG,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,gBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB7E;;;;OAIG;IACG,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,QAAQ,GAAG,iBAAiB,CAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9G;;;;OAIG;IACG,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,GAAE,OAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjF;;;;OAIG;IACG,UAAU,CAAC,CAAC,GAAG,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnF;;;OAGG;IACH,SAAS,CAAC,CAAC,GAAG,GAAG,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,IAAI;IAIpD;;;OAGG;IACH,UAAU,CAAC,OAAO,EAAE,cAAc,GAAG,IAAI;IAIzC;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAIzB;;;OAGG;IACH,kBAAkB,CAAC,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,qBAAqB,GAAG;QAAE,UAAU,CAAC,EAAE,GAAG,CAAA;KAAE,CAAC;IAyB9G;;OAEG;IACG,iBAAiB,CACrB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,UAAe,EACxB,WAAW,GAAE,qBAA0B,GACtC,OAAO,CAAC,qBAAqB,GAAG;QAAE,UAAU,CAAC,EAAE,GAAG,CAAA;KAAE,CAAC;IAMxD;;OAEG;IACG,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAa3E;;;;;;;OAOG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAW,EACnB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAWhB;;OAEG;IACG,aAAa,CACjB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAIhB;;;OAGG;IACG,cAAc,CAClB,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,EACvB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAQhB;;OAEG;IACH,sBAAsB,IAAI;QACxB,oBAAoB,EAAE,OAAO,CAAC;QAC9B,oBAAoB,EAAE,OAAO,CAAC;QAC9B,YAAY,EAAE,OAAO,CAAC;KACvB;IAQD;;;OAGG;IACH,iBAAiB,IAAI,IAAI;IASzB;;;OAGG;IACH,gBAAgB,IAAI,IAAI;IASxB;;OAEG;IACH,cAAc,IAAI,OAAO;IAIzB;;;;;;;;OAQG;IACH,gBAAgB,CAAC,OAAO,GAAE,iBAAsB,GAAG,MAAM,IAAI;IA2D7D;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAIzB;;OAEG;IACH,WAAW,IAAI,IAAI;IAMnB,OAAO,CAAC,wBAAwB;IAqDhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAU7B;;;;;;OAMG;IACG,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;CAqClG;AAED;;GAEG;AACH,qBAAa,UAAU;IACT,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,WAAW;IAE1C;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAMtC;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAU5D;;;;;OAKG;IACG,kBAAkB,CACtB,MAAM,EAAE,MAAM,EACd,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,IAAI,CAAC,cAAc,EAAE,aAAa,GAAG,QAAQ,CAAM,GAC3D,OAAO,CAAC,IAAI,CAAC;IAQhB;;;OAGG;IACG,SAAS,CACb,WAAW,EAAE,MAAM,EACnB,MAAM,GAAE,MAAW,EACnB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAIhB;;OAEG;IACG,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAIhB;;;OAGG;IACG,cAAc,CAClB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,IAAI,CAAC;IAIhB;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,GAAE,OAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7E;;;;OAIG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,GAAE,OAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;OAGG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOtD;;;;OAIG;IACG,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,sBAA2B,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;CAGlG;AAGD,eAAO,MAAM,SAAS,aAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,YAA4B,CAAC"}

package/dist/llm/index.js

@@ -12,6 +12,8 @@ class R1Messaging {
     constructor() {
         this.messageHandlers = new Set();
         this.isInitialized = false;
+        this.pushToTalkCleanup = null;
+        this.sttListening = false;
         this.initializeMessageHandler();
     }
     /**
@@ -48,14 +50,15 @@ class R1Messaging {
      * @param options Additional options
      */
     async searchWeb(query, options = {}) {
+        const { useLocation = false, tag = 'search', ...messageOptions } = options;
         const payload = {
             message: JSON.stringify({
                 query: query,
-                useLocation: false,
-                tag: 'search'
+                useLocation,
+                tag
             }),
             useSerpAPI: true,
-            ...options
+            ...messageOptions
         };
         if (typeof PluginMessageHandler !== 'undefined') {
             PluginMessageHandler.postMessage(JSON.stringify(payload));
@@ -117,6 +120,174 @@ class R1Messaging {
     removeAllHandlers() {
         this.messageHandlers.clear();
     }
+    /**
+     * Wait for the next incoming plugin message.
+     * Useful for request/response flows with timeout handling.
+     */
+    waitForNextMessage(options = {}) {
+        const { timeoutMs = 30000, predicate } = options;
+        return new Promise((resolve, reject) => {
+            let timeoutId = null;
+            const handler = (response) => {
+                if (predicate && !predicate(response))
+                    return;
+                if (timeoutId) {
+                    clearTimeout(timeoutId);
+                }
+                this.offMessage(handler);
+                resolve(response);
+            };
+            this.onMessage(handler);
+            timeoutId = setTimeout(() => {
+                this.offMessage(handler);
+                reject(new Error(`Timed out waiting for plugin message after ${timeoutMs}ms`));
+            }, timeoutMs);
+        });
+    }
+    /**
+     * Send a message to LLM and await the next matching response.
+     */
+    async askLLMWithTimeout(message, options = {}, waitOptions = {}) {
+        const responsePromise = this.waitForNextMessage(waitOptions);
+        await this.askLLM(message, options);
+        return responsePromise;
+    }
+    /**
+     * Ask the LLM to email content to the user.
+     */
+    async emailUser(content, options = {}) {
+        const { instructionPrefix = 'Please email this to the user:', ...messageOptions } = options;
+        await this.sendMessage(`${instructionPrefix} ${content}`, {
+            useLLM: true,
+            wantsJournalEntry: false,
+            ...messageOptions
+        });
+    }
+    /**
+     * Send an image to AI with an optional prompt.
+     *
+     * Common patterns:
+     * - Ask about image: provide a question prompt.
+     * - Transform image: use a prompt like "take a photo and make it ...".
+     * - No-prompt processing: pass empty prompt.
+     */
+    async sendImageToAI(imageBase64, prompt = '', options = {}) {
+        const message = prompt.trim();
+        const normalizedImage = this.normalizeImagePayload(imageBase64);
+        await this.sendMessage(message, {
+            useLLM: true,
+            imageBase64: normalizedImage,
+            ...options
+        });
+    }
+    /**
+     * Ask AI about an image by sending image + prompt.
+     */
+    async askAboutImage(imageBase64, prompt, options = {}) {
+        await this.sendImageToAI(imageBase64, prompt, options);
+    }
+    /**
+     * Transform an image using the common R1 phrasing.
+     * Sends: "take a photo and make it <prompt>" + image.
+     */
+    async transformImage(imageBase64, transformPrompt, options = {}) {
+        const prompt = transformPrompt.trim().length > 0
+            ? `take a photo and make it ${transformPrompt}`
+            : 'take a photo and make it better';
+        await this.sendImageToAI(imageBase64, prompt, options);
+    }
+    /**
+     * Runtime availability checks for messaging-related R1 bridges.
+     */
+    getRuntimeCapabilities() {
+        return {
+            pluginMessageHandler: typeof PluginMessageHandler !== 'undefined',
+            creationVoiceHandler: typeof CreationVoiceHandler !== 'undefined',
+            closeWebView: typeof closeWebView !== 'undefined'
+        };
+    }
+    /**
+     * Start STT listening programmatically.
+     * Equivalent to: `CreationVoiceHandler.postMessage('start')`
+     */
+    startSTTListening() {
+        if (typeof CreationVoiceHandler === 'undefined') {
+            throw new Error('CreationVoiceHandler not available. Make sure you are running in R1 environment.');
+        }
+        CreationVoiceHandler.postMessage('start');
+        this.sttListening = true;
+    }
+    /**
+     * Stop STT listening programmatically.
+     * Equivalent to: `CreationVoiceHandler.postMessage('stop')`
+     */
+    stopSTTListening() {
+        if (typeof CreationVoiceHandler === 'undefined') {
+            throw new Error('CreationVoiceHandler not available. Make sure you are running in R1 environment.');
+        }
+        CreationVoiceHandler.postMessage('stop');
+        this.sttListening = false;
+    }
+    /**
+     * Returns whether STT is currently listening based on SDK-triggered state.
+     */
+    isSTTListening() {
+        return this.sttListening;
+    }
+    /**
+     * Enable push-to-talk using hardware long press events and STT messages.
+     *
+     * Default behavior:
+     * - `longPressStart` => `CreationVoiceHandler.postMessage('start')`
+     * - `longPressEnd` => `CreationVoiceHandler.postMessage('stop')`
+     * - On `sttEnded` with transcript, forwards:
+     *   `{ message: 'process_voice_input', transcript }` via PluginMessageHandler
+     */
+    enablePushToTalk(options = {}) {
+        if (typeof window === 'undefined') {
+            throw new Error('Push-to-talk is only available in browser environments.');
+        }
+        if (typeof CreationVoiceHandler === 'undefined') {
+            throw new Error('CreationVoiceHandler not available. Make sure you are running in R1 environment.');
+        }
+        this.disablePushToTalk();
+        const { startEvent = 'longPressStart', endEvent = 'longPressEnd', processMessage = 'process_voice_input', autoForwardTranscript = true, onTranscript } = options;
+        const handleStart = () => {
+            this.startSTTListening();
+        };
+        const handleEnd = () => {
+            this.stopSTTListening();
+        };
+        const transcriptHandler = (data) => {
+            const type = data.type;
+            const transcript = data.transcript;
+            if (type !== 'sttEnded' || !transcript)
+                return;
+            onTranscript?.(transcript, data);
+            if (autoForwardTranscript && typeof PluginMessageHandler !== 'undefined') {
+                PluginMessageHandler.postMessage(JSON.stringify({
+                    message: processMessage,
+                    transcript
+                }));
+            }
+        };
+        window.addEventListener(startEvent, handleStart);
+        window.addEventListener(endEvent, handleEnd);
+        this.onMessage(transcriptHandler);
+        this.pushToTalkCleanup = () => {
+            window.removeEventListener(startEvent, handleStart);
+            window.removeEventListener(endEvent, handleEnd);
+            this.offMessage(transcriptHandler);
+            this.pushToTalkCleanup = null;
+        };
+        return () => this.disablePushToTalk();
+    }
+    /**
+     * Disable push-to-talk listeners previously configured via enablePushToTalk.
+     */
+    disablePushToTalk() {
+        this.pushToTalkCleanup?.();
+    }
     /**
      * Close the current plugin/webview
      */
@@ -128,6 +299,7 @@ class R1Messaging {
     initializeMessageHandler() {
         if (this.isInitialized || typeof window === 'undefined')
             return;
+        const previousOnPluginMessage = window.onPluginMessage;
         // Set up global message handler
         window.onPluginMessage = (data) => {
             try {
@@ -142,8 +314,16 @@ class R1Messaging {
                         parsedData = data.data;
                     }
                 }
+                const objectData = parsedData && typeof parsedData === 'object' ? parsedData : null;
+                const typeFromParsed = typeof objectData?.type === 'string' ? objectData.type : undefined;
+                const transcriptFromParsed = typeof objectData?.transcript === 'string' ? objectData.transcript : undefined;
                 // Call all registered handlers
-                const enhancedData = { ...data, parsedData };
+                const enhancedData = {
+                    ...data,
+                    type: data.type ?? typeFromParsed,
+                    transcript: data.transcript ?? transcriptFromParsed,
+                    parsedData
+                };
                 this.messageHandlers.forEach(handler => {
                     try {
                         handler(enhancedData);
@@ -152,6 +332,14 @@ class R1Messaging {
                         console.error('Error in message handler:', error);
                     }
                 });
+                if (previousOnPluginMessage) {
+                    try {
+                        previousOnPluginMessage(data);
+                    }
+                    catch (error) {
+                        console.error('Error in pre-existing onPluginMessage handler:', error);
+                    }
+                }
             }
             catch (error) {
                 console.error('Error processing plugin message:', error);
@@ -159,6 +347,61 @@ class R1Messaging {
         };
         this.isInitialized = true;
     }
+    /**
+     * Normalize image payload for R1 bridge compatibility.
+     * Accepts either full data URL or raw base64 and returns a data URL string.
+     */
+    normalizeImagePayload(image) {
+        const value = image.trim();
+        if (value.startsWith('data:image/')) {
+            return value;
+        }
+        // Default to jpeg wrapper when only raw base64 is provided.
+        return `data:image/jpeg;base64,${value}`;
+    }
+    /**
+     * Generate audio file from text-to-speech (browser only)
+     * Uses Web Speech API to synthesize speech and capture as audio blob
+     * @param text Text to convert to audio
+     * @param options Speech synthesis options
+     * @returns Promise resolving to audio blob or null if not supported
+     */
+    async textToSpeechAudio(text, options = {}) {
+        // Only works in browser environment
+        if (typeof window === 'undefined' || !window.speechSynthesis) {
+            throw new Error('Text-to-speech audio generation only available in browser with Web Speech API support');
+        }
+        return new Promise((resolve, reject) => {
+            try {
+                // Create speech utterance
+                const utterance = new SpeechSynthesisUtterance(text);
+                // Apply options
+                if (options.voice)
+                    utterance.voice = options.voice;
+                if (options.rate !== undefined)
+                    utterance.rate = options.rate;
+                if (options.pitch !== undefined)
+                    utterance.pitch = options.pitch;
+                if (options.volume !== undefined)
+                    utterance.volume = options.volume;
+                // Listen for speech end
+                utterance.onend = () => {
+                    // Currently, we can't capture the audio output from speech synthesis
+                    // This would require advanced techniques like AudioWorklet or WebRTC
+                    // For now, we just speak and return null
+                    resolve(null);
+                };
+                utterance.onerror = (error) => {
+                    reject(new Error(`Speech synthesis failed: ${error.error}`));
+                };
+                // Speak the text
+                window.speechSynthesis.speak(utterance);
+            }
+            catch (error) {
+                reject(error);
+            }
+        });
+    }
 }
 exports.R1Messaging = R1Messaging;
 /**
@@ -187,6 +430,39 @@ class LLMHelpers {
         message += ' Please respond with a JSON analysis.';
         await this.messaging.askLLMJSON(message);
     }
+    /**
+     * Analyze a base64-encoded image with the LLM.
+     * @param prompt Analysis prompt
+     * @param imageBase64 Base64 image string (without data URL prefix)
+     * @param options Additional message options
+     */
+    async analyzeImageBase64(prompt, imageBase64, options = {}) {
+        await this.messaging.sendMessage(prompt, {
+            useLLM: true,
+            imageBase64,
+            ...options
+        });
+    }
+    /**
+     * Send an image to AI with optional prompt.
+     * Empty prompt triggers no-prompt image processing.
+     */
+    async imageToAI(imageBase64, prompt = '', options = {}) {
+        await this.messaging.sendImageToAI(imageBase64, prompt, options);
+    }
+    /**
+     * Ask AI about an image (image + prompt).
+     */
+    async askImage(imageBase64, prompt, options = {}) {
+        await this.messaging.askAboutImage(imageBase64, prompt, options);
+    }
+    /**
+     * Transform an image with phrasing:
+     * "take a photo and make it <prompt>".
+     */
+    async transformPhoto(imageBase64, prompt, options = {}) {
+        await this.messaging.transformImage(imageBase64, prompt, options);
+    }
     /**
      * Ask LLM to perform a task and speak the result
      * @param task Task description
@@ -211,6 +487,14 @@ class LLMHelpers {
         await this.messaging.askLLMJSON(`Given this UI context: "${context}", provide suggestions for user actions. ` +
             `Respond with JSON format: {"suggestions": [{"action": "action_name", "description": "description"}]}`);
     }
+    /**
+     * Generate audio file from text-to-speech (browser only)
+     * @param text Text to convert to audio
+     * @param options Speech synthesis options
+     */
+    async textToSpeechAudio(text, options = {}) {
+        return this.messaging.textToSpeechAudio(text, options);
+    }
 }
 exports.LLMHelpers = LLMHelpers;
 // Export singleton instances