veryfront 0.1.438 → 0.1.439

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/esm/deno.js +1 -1
  2. package/esm/extensions/ext-anthropic/src/anthropic-provider.d.ts.map +1 -1
  3. package/esm/extensions/ext-anthropic/src/anthropic-provider.js +23 -5
  4. package/esm/extensions/ext-google/src/google-provider.d.ts.map +1 -1
  5. package/esm/extensions/ext-google/src/google-provider.js +22 -2
  6. package/esm/extensions/ext-openai/src/openai-provider.d.ts.map +1 -1
  7. package/esm/extensions/ext-openai/src/openai-provider.js +23 -2
  8. package/esm/src/agent/runtime/text-generation-runtime-message-converter.d.ts.map +1 -1
  9. package/esm/src/agent/runtime/text-generation-runtime-message-converter.js +38 -2
  10. package/esm/src/agent/runtime/text-generation-runtime-message-types.d.ts +7 -1
  11. package/esm/src/agent/runtime/text-generation-runtime-message-types.d.ts.map +1 -1
  12. package/esm/src/agent/runtime-message-file-url-refresh.d.ts.map +1 -1
  13. package/esm/src/agent/runtime-message-file-url-refresh.js +64 -10
  14. package/esm/src/chat/conversation.d.ts.map +1 -1
  15. package/esm/src/chat/conversation.js +8 -5
  16. package/esm/src/provider/runtime-loader.d.ts +14 -1
  17. package/esm/src/provider/runtime-loader.d.ts.map +1 -1
  18. package/esm/src/provider/runtime-loader.js +19 -1
  19. package/esm/src/runtime/runtime-bridge.d.ts.map +1 -1
  20. package/esm/src/runtime/runtime-bridge.js +3 -1
  21. package/esm/src/utils/version-constant.d.ts +1 -1
  22. package/esm/src/utils/version-constant.js +1 -1
  23. package/package.json +1 -1
  24. package/src/deno.js +1 -1
  25. package/src/extensions/ext-anthropic/src/anthropic-provider.ts +28 -5
  26. package/src/extensions/ext-google/src/google-provider.ts +27 -2
  27. package/src/extensions/ext-openai/src/openai-provider.ts +29 -2
  28. package/src/src/agent/runtime/text-generation-runtime-message-converter.ts +41 -2
  29. package/src/src/agent/runtime/text-generation-runtime-message-types.ts +8 -1
  30. package/src/src/agent/runtime-message-file-url-refresh.ts +78 -12
  31. package/src/src/chat/conversation.ts +15 -9
  32. package/src/src/provider/runtime-loader.ts +46 -3
  33. package/src/src/runtime/runtime-bridge.ts +10 -2
  34. package/src/src/utils/version-constant.ts +1 -1
package/esm/deno.js CHANGED
@@ -1,6 +1,6 @@
1
1
  export default {
2
2
  "name": "veryfront",
3
- "version": "0.1.438",
3
+ "version": "0.1.439",
4
4
  "license": "Apache-2.0",
5
5
  "nodeModulesDir": "auto",
6
6
  "workspace": [
package/esm/extensions/ext-anthropic/src/anthropic-provider.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"anthropic-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-anthropic/src/anthropic-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EACL,kBAAkB,EAIlB,aAAa,EACb,UAAU,EACV,iBAAiB,EAEjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAOpB,+BAA+B,EAE/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAG/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AA0hCD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,EAAE,MAAM,GACd,YAAY,CAyEd;AAED,qBAAa,iBAAkB,YAAW,UAAU;IAClD,QAAQ,CAAC,EAAE,eAAe;IAE1B,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;CAYrE"}
1
+ {"version":3,"file":"anthropic-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-anthropic/src/anthropic-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACnE,OAAO,EACL,kBAAkB,EAIlB,aAAa,EACb,UAAU,EACV,iBAAiB,EAEjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAMpB,+BAA+B,EAE/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAG/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAkjCD,wBAAgB,2BAA2B,CACzC,MAAM,EAAE,sBAAsB,EAC9B,OAAO,EAAE,MAAM,GACd,YAAY,CAyEd;AAED,qBAAa,iBAAkB,YAAW,UAAU;IAClD,QAAQ,CAAC,EAAE,eAAe;IAE1B,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;CAYrE"}
package/esm/extensions/ext-anthropic/src/anthropic-provider.js CHANGED
@@ -7,7 +7,7 @@
7
7
  *
8
8
  * @module extensions/ext-anthropic/anthropic-provider
9
9
  */
10
- import { buildProviderError, createAnthropicRequestInit, createWarningCollector, getAnthropicMessagesUrl, isNumberArray, mergeUsage, parseRetryAfterMs, parseSseChunk, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, readTextParts, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, unwrapToolInputSchema, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
10
+ import { buildProviderError, createAnthropicRequestInit, createWarningCollector, getAnthropicMessagesUrl, isNumberArray, mergeUsage, parseRetryAfterMs, parseSseChunk, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, unwrapToolInputSchema, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
11
11
  export { buildProviderError, isNumberArray, mergeUsage, parseRetryAfterMs, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, TOOL_INPUT_PENDING_THRESHOLD_MS, withToolInputStatusTransitions, };
12
12
  // ---------------------------------------------------------------------------
13
13
  // Anthropic helper functions
@@ -92,6 +92,27 @@ function pushAnthropicUserContent(messages, content) {
92
92
  content,
93
93
  });
94
94
  }
95
+ function toAnthropicUserContent(parts) {
96
+ const content = [];
97
+ for (const part of parts) {
98
+ if (part.type === "text") {
99
+ if (part.text.length > 0) {
100
+ content.push({ type: "text", text: part.text });
101
+ }
102
+ continue;
103
+ }
104
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
105
+ content.push({
106
+ type: "image",
107
+ source: {
108
+ type: "url",
109
+ url: part.url,
110
+ },
111
+ });
112
+ }
113
+ }
114
+ return content;
115
+ }
95
116
  /**
96
117
  * Resolves a {@link ProviderCacheTtl} into Anthropic's `cache_control` shape.
97
118
  *
@@ -119,10 +140,7 @@ function toAnthropicMessages(prompt, systemCacheControl) {
119
140
  }
120
141
  break;
121
142
  case "user":
122
- pushAnthropicUserContent(messages, [{
123
- type: "text",
124
- text: readTextParts(message.content),
125
- }]);
143
+ pushAnthropicUserContent(messages, toAnthropicUserContent(message.content));
126
144
  break;
127
145
  case "assistant":
128
146
  messages.push({
package/esm/extensions/ext-google/src/google-provider.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"google-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-google/src/google-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAOpB,+BAA+B,EAC/B,qBAAqB,EACrB,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAS/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,qBAAqB,EACrB,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAwoBD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAkEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CA6ClB;AAED,qBAAa,cAAe,YAAW,UAAU;IAC/C,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;IAYpE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,gBAAgB;CAW7E"}
1
+ {"version":3,"file":"google-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-google/src/google-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAMpB,+BAA+B,EAC/B,qBAAqB,EACrB,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAS/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,qBAAqB,EACrB,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAkqBD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAkEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CA6ClB;AAED,qBAAa,cAAe,YAAW,UAAU;IAC/C,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;IAYpE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,gBAAgB;CAW7E"}
package/esm/extensions/ext-google/src/google-provider.js CHANGED
@@ -6,7 +6,7 @@
6
6
  *
7
7
  * @module extensions/ext-google/google-provider
8
8
  */
9
- import { buildProviderError, createGoogleRequestInit, createWarningCollector, getGoogleEmbeddingUrl, getGoogleGenerateContentUrl, getGoogleStreamGenerateContentUrl, isNumberArray, mergeUsage, parseRetryAfterMs, parseSseChunk, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, readTextParts, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, unwrapToolInputSchema, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
9
+ import { buildProviderError, createGoogleRequestInit, createWarningCollector, getGoogleEmbeddingUrl, getGoogleGenerateContentUrl, getGoogleStreamGenerateContentUrl, isNumberArray, mergeUsage, parseRetryAfterMs, parseSseChunk, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, unwrapToolInputSchema, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
10
10
  // Re-export error classes so extension tests can import them from this module
11
11
  // and from `veryfront/provider/shared` interchangeably.
12
12
  export { buildProviderError, isNumberArray, mergeUsage, parseRetryAfterMs, parseSseChunk, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, TOOL_INPUT_PENDING_THRESHOLD_MS, unwrapToolInputSchema, withToolInputStatusTransitions, };
@@ -84,7 +84,7 @@ function toGoogleContents(prompt) {
84
84
  case "user":
85
85
  contents.push({
86
86
  role: "user",
87
- parts: [{ text: readTextParts(message.content) }],
87
+ parts: toGoogleUserParts(message.content),
88
88
  });
89
89
  break;
90
90
  case "assistant": {
@@ -133,6 +133,26 @@ function toGoogleContents(prompt) {
133
133
  contents,
134
134
  };
135
135
  }
136
+ function toGoogleUserParts(parts) {
137
+ const content = [];
138
+ for (const part of parts) {
139
+ if (part.type === "text") {
140
+ if (part.text.length > 0) {
141
+ content.push({ text: part.text });
142
+ }
143
+ continue;
144
+ }
145
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
146
+ content.push({
147
+ fileData: {
148
+ mimeType: part.mediaType,
149
+ fileUri: part.url,
150
+ },
151
+ });
152
+ }
153
+ }
154
+ return content;
155
+ }
136
156
  function toGoogleTools(tools) {
137
157
  if (!tools) {
138
158
  return undefined;
package/esm/extensions/ext-openai/src/openai-provider.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"openai-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-openai/src/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAOpB,+BAA+B,EAG/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAI/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAoqCD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CAuClB;AAED,qBAAa,cAAe,YAAW,UAAU;IAC/C,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;IAYpE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,gBAAgB;IAY5E,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;CAWzE"}
1
+ {"version":3,"file":"openai-provider.d.ts","sourceRoot":"","sources":["../../../../src/extensions/ext-openai/src/openai-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAChG,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,gCAAgC,CAAC;AACrF,OAAO,EACL,kBAAkB,EAMlB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EAMpB,+BAA+B,EAG/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAI/C,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,+BAA+B,EAC/B,8BAA8B,GAC/B,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAgsCD,wBAAgB,wBAAwB,CACtC,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,YAAY,CAsEd;AAED,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,mBAAmB,EAC3B,OAAO,EAAE,MAAM,GACd,gBAAgB,CAuClB;AAED,qBAAa,cAAe,YAAW,UAAU;IAC/C,QAAQ,CAAC,EAAE,YAAY;IAEvB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;IAYpE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,gBAAgB;IAY5E,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,YAAY;CAWzE"}
package/esm/extensions/ext-openai/src/openai-provider.js CHANGED
@@ -7,7 +7,7 @@
7
7
  *
8
8
  * @module extensions/ext-openai/openai-provider
9
9
  */
10
- import { buildProviderError, createOpenAIRequestInit, createWarningCollector, getOpenAIChatCompletionsUrl, getOpenAIEmbeddingUrl, getOpenAIResponsesUrl, isNumberArray, mergeUsage, parseRetryAfterMs, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, readTextParts, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, toOpenAICompatibleMessages, toOpenAICompatibleTools, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
10
+ import { buildProviderError, createOpenAIRequestInit, createWarningCollector, getOpenAIChatCompletionsUrl, getOpenAIEmbeddingUrl, getOpenAIResponsesUrl, isNumberArray, mergeUsage, parseRetryAfterMs, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, readProviderOptions, readRecord, requestJson, requestStream, stringifyJsonValue, TOOL_INPUT_PENDING_THRESHOLD_MS, toOpenAICompatibleMessages, toOpenAICompatibleTools, withToolInputStatusTransitions, } from "../../../src/provider/shared/index.js";
11
11
  // Re-export error classes so extension tests can import from this module.
12
12
  export { buildProviderError, isNumberArray, mergeUsage, parseRetryAfterMs, ProviderError, ProviderOverloadedError, ProviderQuotaError, ProviderRateLimitError, ProviderRequestError, TOOL_INPUT_PENDING_THRESHOLD_MS, withToolInputStatusTransitions, };
13
13
  // ---------------------------------------------------------------------------
@@ -480,7 +480,7 @@ function toOpenAIResponsesInput(prompt) {
480
480
  case "user":
481
481
  input.push({
482
482
  role: "user",
483
- content: [{ type: "input_text", text: readTextParts(message.content) }],
483
+ content: toOpenAIResponsesUserContent(message.content),
484
484
  });
485
485
  break;
486
486
  case "assistant": {
@@ -543,6 +543,27 @@ function toOpenAIResponsesInput(prompt) {
543
543
  input,
544
544
  };
545
545
  }
546
+ function toOpenAIResponsesUserContent(parts) {
547
+ const content = [];
548
+ for (const part of parts) {
549
+ if (part.type === "text") {
550
+ if (part.text.length > 0) {
551
+ content.push({ type: "input_text", text: part.text });
552
+ }
553
+ continue;
554
+ }
555
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
556
+ content.push({ type: "input_image", image_url: part.url, detail: "auto" });
557
+ continue;
558
+ }
559
+ content.push({
560
+ type: "input_file",
561
+ file_url: part.url,
562
+ ...(part.filename ? { filename: part.filename } : {}),
563
+ });
564
+ }
565
+ return content;
566
+ }
546
567
  /**
547
568
  * Tools on the Responses API differ from Chat Completions: instead of
548
569
  * `{ type: "function", function: { name, parameters } }` the function
package/esm/src/agent/runtime/text-generation-runtime-message-converter.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"text-generation-runtime-message-converter.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/text-generation-runtime-message-converter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAEV,4BAA4B,EAI7B,MAAM,4CAA4C,CAAC;AAEpD,OAAO,EAGL,KAAK,OAAO,EAGb,MAAM,aAAa,CAAC;AAsDrB;;GAEG;AACH,wBAAgB,qCAAqC,CAAC,GAAG,EAAE,OAAO,GAAG,4BAA4B,CAyEhG;AAgBD;;GAEG;AACH,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,OAAO,EAAE,GAClB,4BAA4B,EAAE,CAqChC"}
1
+ {"version":3,"file":"text-generation-runtime-message-converter.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/text-generation-runtime-message-converter.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAGV,4BAA4B,EAI7B,MAAM,4CAA4C,CAAC;AAEpD,OAAO,EAGL,KAAK,OAAO,EAGb,MAAM,aAAa,CAAC;AA0ErB;;GAEG;AACH,wBAAgB,qCAAqC,CAAC,GAAG,EAAE,OAAO,GAAG,4BAA4B,CA2FhG;AAgBD;;GAEG;AACH,wBAAgB,sCAAsC,CACpD,QAAQ,EAAE,OAAO,EAAE,GAClB,4BAA4B,EAAE,CAqChC"}
package/esm/src/agent/runtime/text-generation-runtime-message-converter.js CHANGED
@@ -52,6 +52,25 @@ function getUserTextWithAttachmentContext(parts) {
52
52
  ? text
53
53
  : appendReadableAttachmentContext(text, buildAttachmentContextFromParts(parts));
54
54
  }
55
+ function getUserFileParts(parts) {
56
+ return parts.flatMap((part) => {
57
+ const type = getStringPartField(part, "type");
58
+ if (type !== "file" && type !== "image")
59
+ return [];
60
+ const mediaType = getStringPartField(part, "mediaType");
61
+ const url = getStringPartField(part, "url");
62
+ if (!mediaType || !url || url.startsWith("data:"))
63
+ return [];
64
+ return [{
65
+ type,
66
+ mediaType,
67
+ url,
68
+ ...(getStringPartField(part, "filename")
69
+ ? { filename: getStringPartField(part, "filename") }
70
+ : {}),
71
+ }];
72
+ });
73
+ }
55
74
  /**
56
75
  * Convert a veryfront Message to the current text-generation runtime message format.
57
76
  */
@@ -62,8 +81,25 @@ export function convertToTextGenerationRuntimeMessage(msg) {
62
81
  return { role: "system", content: text };
63
82
  }
64
83
  case "user": {
65
- const text = getUserTextWithAttachmentContext(msg.parts);
66
- return { role: "user", content: text };
84
+ const fileParts = getUserFileParts(msg.parts);
85
+ if (fileParts.length === 0) {
86
+ const text = getUserTextWithAttachmentContext(msg.parts);
87
+ return { role: "user", content: text };
88
+ }
89
+ const text = getTextFromParts(msg.parts);
90
+ const attachmentContext = text.includes("<uploaded_files>")
91
+ ? ""
92
+ : buildAttachmentContextFromParts(msg.parts);
93
+ return {
94
+ role: "user",
95
+ content: [
96
+ ...(text.length > 0 ? [{ type: "text", text }] : []),
97
+ ...fileParts,
98
+ ...(attachmentContext.length > 0
99
+ ? [{ type: "text", text: attachmentContext.trimStart() }]
100
+ : []),
101
+ ],
102
+ };
67
103
  }
68
104
  case "assistant": {
69
105
  const content = [];
package/esm/src/agent/runtime/text-generation-runtime-message-types.d.ts CHANGED
@@ -9,6 +9,12 @@ export interface TextGenerationRuntimeTextPart {
9
9
  type: "text";
10
10
  text: string;
11
11
  }
12
+ export interface TextGenerationRuntimeFilePart {
13
+ type: "file" | "image";
14
+ mediaType: string;
15
+ url: string;
16
+ filename?: string;
17
+ }
12
18
  export interface TextGenerationRuntimeToolCallPart {
13
19
  type: "tool-call";
14
20
  toolCallId: string;
@@ -30,7 +36,7 @@ export interface TextGenerationRuntimeSystemMessage {
30
36
  }
31
37
  export interface TextGenerationRuntimeUserMessage {
32
38
  role: "user";
33
- content: string;
39
+ content: string | Array<TextGenerationRuntimeTextPart | TextGenerationRuntimeFilePart>;
34
40
  }
35
41
  export interface TextGenerationRuntimeAssistantMessage {
36
42
  role: "assistant";
package/esm/src/agent/runtime/text-generation-runtime-message-types.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"text-generation-runtime-message-types.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/text-generation-runtime-message-types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iCAAiC;IAChD,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,mCAAmC;IAClD,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,OAAO,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qCAAqC;IACpD,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,KAAK,CAAC,6BAA6B,GAAG,iCAAiC,CAAC,CAAC;CACnF;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,mCAAmC,EAAE,CAAC;CAChD;AAED,MAAM,MAAM,4BAA4B,GACpC,kCAAkC,GAClC,gCAAgC,GAChC,qCAAqC,GACrC,gCAAgC,CAAC"}
1
+ {"version":3,"file":"text-generation-runtime-message-types.d.ts","sourceRoot":"","sources":["../../../../src/src/agent/runtime/text-generation-runtime-message-types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,6BAA6B;IAC5C,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iCAAiC;IAChD,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,mCAAmC;IAClD,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,OAAO,CAAC;KAChB,CAAC;CACH;AAED,MAAM,WAAW,kCAAkC;IACjD,IAAI,EAAE,QAAQ,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,KAAK,CAAC,6BAA6B,GAAG,6BAA6B,CAAC,CAAC;CACxF;AAED,MAAM,WAAW,qCAAqC;IACpD,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,KAAK,CAAC,6BAA6B,GAAG,iCAAiC,CAAC,CAAC;CACnF;AAED,MAAM,WAAW,gCAAgC;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,mCAAmC,EAAE,CAAC;CAChD;AAED,MAAM,MAAM,4BAA4B,GACpC,kCAAkC,GAClC,gCAAgC,GAChC,qCAAqC,GACrC,gCAAgC,CAAC"}
package/esm/src/agent/runtime-message-file-url-refresh.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runtime-message-file-url-refresh.d.ts","sourceRoot":"","sources":["../../../src/src/agent/runtime-message-file-url-refresh.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,aAAa,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE5F,MAAM,MAAM,2BAA2B,GAAG;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CACnC,KAAK,EAAE,2BAA2B,KAC/B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;AAEjC,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,SAAS,aAAa,EAAE,EAClC,cAAc,EAAE,sBAAsB,GACrC,OAAO,CAAC,aAAa,EAAE,CAAC,CAgC1B"}
1
+ {"version":3,"file":"runtime-message-file-url-refresh.d.ts","sourceRoot":"","sources":["../../../src/src/agent/runtime-message-file-url-refresh.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE5E,MAAM,MAAM,2BAA2B,GAAG;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,oBAAoB,CAAC;IAC3B,OAAO,EAAE,aAAa,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG,CACnC,KAAK,EAAE,2BAA2B,KAC/B,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;AAEjC,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,SAAS,aAAa,EAAE,EAClC,cAAc,EAAE,sBAAsB,GACrC,OAAO,CAAC,aAAa,EAAE,CAAC,CAqC1B"}
package/esm/src/agent/runtime-message-file-url-refresh.js CHANGED
@@ -1,28 +1,82 @@
1
1
  export async function resolveRuntimeMessageFileUrls(messages, resolveFileUrl) {
2
2
  const urlByUploadId = new Map();
3
3
  return Promise.all(messages.map(async (message) => {
4
- if (!message.parts.some((part) => part.type === "file" && part.uploadId)) {
4
+ if (!message.parts.some((part) => getUploadId(part))) {
5
5
  return message;
6
6
  }
7
7
  const parts = await Promise.all(message.parts.map(async (part) => {
8
- if (!isFilePartWithUpload(part))
8
+ const uploadId = getUploadId(part);
9
+ if (!uploadId)
9
10
  return part;
10
- let urlPromise = urlByUploadId.get(part.uploadId);
11
+ let urlPromise = urlByUploadId.get(uploadId);
11
12
  if (!urlPromise) {
12
- urlPromise = resolveFileUrl({ uploadId: part.uploadId, part, message });
13
- urlByUploadId.set(part.uploadId, urlPromise);
13
+ urlPromise = resolveFileUrl({
14
+ uploadId,
15
+ part: toResolverPart(part, uploadId),
16
+ message,
17
+ });
18
+ urlByUploadId.set(uploadId, urlPromise);
14
19
  }
15
20
  const signedUrl = await urlPromise;
16
- if (!signedUrl || signedUrl === part.url)
17
- return part;
21
+ if (!signedUrl)
22
+ return normalizeUploadedFilePart(part, uploadId);
18
23
  return {
19
- ...part,
24
+ ...normalizeUploadedFilePart(part, uploadId),
20
25
  url: signedUrl,
21
26
  };
22
27
  }));
23
28
  return { ...message, parts };
24
29
  }));
25
30
  }
26
- function isFilePartWithUpload(part) {
27
- return part.type === "file" && typeof part.uploadId === "string" && part.uploadId.length > 0;
31
+ function isRecord(value) {
32
+ return typeof value === "object" && value !== null && !Array.isArray(value);
33
+ }
34
+ function getStringField(part, key) {
35
+ if (!isRecord(part))
36
+ return undefined;
37
+ const value = part[key];
38
+ return typeof value === "string" && value.length > 0 ? value : undefined;
39
+ }
40
+ function getUploadId(part) {
41
+ if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
42
+ return undefined;
43
+ }
44
+ return getStringField(part, "uploadId") ?? getStringField(part, "upload_id");
45
+ }
46
+ function getMediaType(part) {
47
+ return getStringField(part, "mediaType") ?? getStringField(part, "media_type");
48
+ }
49
+ function normalizeUploadedFilePart(part, uploadId) {
50
+ if (!isRecord(part))
51
+ return part;
52
+ const partRecord = part;
53
+ const partType = partRecord.type;
54
+ if (partType !== "file" && partType !== "image")
55
+ return part;
56
+ const mediaType = getMediaType(part);
57
+ const url = getStringField(part, "url");
58
+ if (!mediaType || !url)
59
+ return part;
60
+ const filename = getStringField(part, "filename");
61
+ const uploadPath = getStringField(part, "uploadPath") ?? getStringField(part, "upload_path");
62
+ return {
63
+ type: partType === "image" ? "image" : "file",
64
+ mediaType,
65
+ url,
66
+ ...(filename ? { filename } : {}),
67
+ uploadId,
68
+ ...(uploadPath ? { uploadPath } : {}),
69
+ };
70
+ }
71
+ function toResolverPart(part, uploadId) {
72
+ const normalized = normalizeUploadedFilePart(part, uploadId);
73
+ if (isRecord(normalized) && normalized.type === "file") {
74
+ return normalized;
75
+ }
76
+ return {
77
+ type: "file",
78
+ mediaType: getMediaType(part) ?? "application/octet-stream",
79
+ url: getStringField(part, "url") ?? "",
80
+ uploadId,
81
+ };
28
82
  }
package/esm/src/chat/conversation.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../../../src/src/chat/conversation.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAiDzF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BAW5B,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,sBAAsB;;;;;;EAMjC,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,mBAAmB;;;;;;;;EAQ9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;iBAehC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAkB3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,KAAK,UAAU,GAAG,OAAO,CAAC,iBAAiB,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAGpF,eAAO,MAAM,YAAY,QACuD,CAAC;AAEjF,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,KAAK,IAAI,MAAM,CAExE;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAG1D;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAiB9F;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEzE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMtF;AAoBD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAQvD;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,GACtB,IAAI,IAAI,iBAAiB,GAAG;IAAE,IAAI,EAAE,QAAQ,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,CAEvE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,iBAAiB,GAAG,IAAI,IAAI,UAAU,CAMxE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS,CAOlE;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,EAAE,EACpB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,GACA,IAAI,CAiCN;AA2BD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,aAAa,GAAG,WAAW,EAAE,CAiEtF;AAOD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAEtE;AAED,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,aAAa,GAAG,aAAa,CAEtF;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,MAAM,GAChB,aAAa,CAaf;AA8BD,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAOxE;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAEhE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,iBAAiB,CAE1E;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAoB5E;AAyaD,wBAAgB,wCAAwC,CACtD,QAAQ,EAAE,aAAa,EAAE,GACxB,oBAAoB,EAAE,CAaxB;AAED;;GAEG;AACH,eAAO,MAAM,gCAAgC,iDAA2C,CAAC"}
1
+ {"version":3,"file":"conversation.d.ts","sourceRoot":"","sources":["../../../src/src/chat/conversation.ts"],"names":[],"mappings":"AAAA,OAAO,yBAAyB,CAAC;AACjC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAiDzF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;2BAW5B,CAAC;AAEH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,sBAAsB;;;;;;EAMjC,CAAC;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,eAAO,MAAM,mBAAmB;;;;;;;;EAQ9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;;;;;iBAehC,CAAC;AAEH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAkB3B,CAAC;AAEH,MAAM,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,aAAa,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;IAChB,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,KAAK,UAAU,GAAG,OAAO,CAAC,iBAAiB,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAGpF,eAAO,MAAM,YAAY,QACuD,CAAC;AAEjF,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,KAAK,IAAI,MAAM,CAExE;AAED,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAG1D;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,GAAG,WAAW,GAAG,OAAO,CAiB9F;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEzE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMtF;AAoBD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAQvD;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,iBAAiB,GACtB,IAAI,IAAI,iBAAiB,GAAG;IAAE,IAAI,EAAE,QAAQ,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,OAAO,CAAA;CAAE,CAEvE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,iBAAiB,GAAG,IAAI,IAAI,UAAU,CAMxE;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,GAAG,SAAS,CAOlE;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,WAAW,EAAE,EACpB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;IACJ,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB,GACA,IAAI,CAiCN;AA2BD,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,aAAa,GAAG,WAAW,EAAE,CAiEtF;AAOD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAEtE;AAED,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,aAAa,GAAG,aAAa,CAEtF;AAED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,aAAa,EACtB,SAAS,EAAE,MAAM,GAChB,aAAa,CAaf;AA8BD,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAOpE;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAOxE;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAEhE;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,iBAAiB,CAE1E;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,CAoB5E;AA+aD,wBAAgB,wCAAwC,CACtD,QAAQ,EAAE,aAAa,EAAE,GACxB,oBAAoB,EAAE,CAaxB;AAED;;GAEG;AACH,eAAO,MAAM,gCAAgC,iDAA2C,CAAC"}
package/esm/src/chat/conversation.js CHANGED
@@ -389,19 +389,22 @@ function toJsonValue(value) {
389
389
  return JSON.stringify(value);
390
390
  }
391
391
  function getFilePart(part) {
392
- if (!isRecord(part) || part.type !== "file") {
392
+ if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
393
393
  return null;
394
394
  }
395
- const mediaType = getNonEmptyStringField(part, "mediaType");
395
+ const mediaType = getNonEmptyStringField(part, "mediaType") ??
396
+ getNonEmptyStringField(part, "media_type");
396
397
  const data = getNonEmptyStringField(part, "url");
397
398
  if (!mediaType || !data) {
398
399
  return null;
399
400
  }
400
401
  const filename = getNonEmptyStringField(part, "filename");
401
- const uploadId = getNonEmptyStringField(part, "uploadId");
402
- const uploadPath = getNonEmptyStringField(part, "uploadPath");
402
+ const uploadId = getNonEmptyStringField(part, "uploadId") ??
403
+ getNonEmptyStringField(part, "upload_id");
404
+ const uploadPath = getNonEmptyStringField(part, "uploadPath") ??
405
+ getNonEmptyStringField(part, "upload_path");
403
406
  return {
404
- type: "file",
407
+ type: part.type === "image" ? "image" : "file",
405
408
  mediaType,
406
409
  data,
407
410
  url: data,
package/esm/src/provider/runtime-loader.d.ts CHANGED
@@ -16,6 +16,11 @@ export type RuntimePromptMessage = {
16
16
  content: Array<{
17
17
  type: "text";
18
18
  text: string;
19
+ } | {
20
+ type: "image" | "file";
21
+ mediaType: string;
22
+ url: string;
23
+ filename?: string;
19
24
  }>;
20
25
  } | {
21
26
  role: "assistant";
@@ -62,7 +67,15 @@ export type OpenAICompatibleChatMessage = {
62
67
  content: string;
63
68
  } | {
64
69
  role: "user";
65
- content: string;
70
+ content: string | Array<{
71
+ type: "text";
72
+ text: string;
73
+ } | {
74
+ type: "image_url";
75
+ image_url: {
76
+ url: string;
77
+ };
78
+ }>;
66
79
  } | {
67
80
  role: "assistant";
68
81
  content: string | null;
package/esm/src/provider/runtime-loader.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runtime-loader.d.ts","sourceRoot":"","sources":["../../../src/src/provider/runtime-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,kDAAkD,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACnF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACd,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAClE,OAAO,EACL,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAE/C,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,+BAA+B,EAAE,8BAA8B,EAAE,CAAC;AAC3E,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,GACd,CAAC;AACF,YAAY,EAAE,YAAY,EAAE,CAAC;AAE7B,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CAAE,GAChE;IACA,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,KAAK,CACV;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC9B;QACA,IAAI,EAAE,WAAW,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,OAAO,CAAC;QACf,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,GACC;QAOA,IAAI,EAAE,WAAW,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CACF,CAAC;CACH,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,aAAa,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,OAAO,CAAA;SAAE,CAAC;KAC1C,CAAC,CAAC;CACJ,CAAC;AACJ,KAAK,qBAAqB,GACtB;IACA,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB,GACC;IACA,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B,CAAC;AAkLJ,MAAM,MAAM,2BAA2B,GACnC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACjC;IACA,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,CAAC,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,UAAU,CAAC;QACjB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC;KACH,CAAC,CAAC;CACJ,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AACJ,MAAM,MAAM,2BAA2B,GAAG;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,2BAA2B,EAAE,CAAC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE;QACf,aAAa,CAAC,EAAE,OAAO,CAAC;KACzB,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,UAAU,CAAC;QACjB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,UAAU,EAAE,OAAO,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;KACH,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AACF;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;GAIG;AACH,KAAK,gBAAgB,GAAG;IACtB,IAAI,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAC;IACrC,KAAK,IAAI,eAAe,EAAE,CAAC;CAC5B,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,gBAAgB,CAUzD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMzD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAQnF;AAED,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,oBAAoB,EAAE,GAC7B,2BAA2B,EAAE,CA0D/B;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,qBAAqB,EAAE,GAAG,SAAS,GACzC,2BAA2B,CAAC,OAAO,CAAC,GAAG,SAAS,CAmBlD;AAED,wBAAgB,mBAAmB,CACjC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EACpD,GAAG,aAAa,EAAE,MAAM,EAAE,GACzB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAezB;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,OAAO,GAAG,OAAO,CAOnE"}
1
+ {"version":3,"file":"runtime-loader.d.ts","sourceRoot":"","sources":["../../../src/src/provider/runtime-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,kDAAkD,CAAC;AACjF,OAAO,EAAE,UAAU,EAAE,KAAK,YAAY,EAAE,MAAM,oCAAoC,CAAC;AACnF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AACtE,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACd,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,sCAAsC,CAAC;AAClE,OAAO,EACL,+BAA+B,EAC/B,8BAA8B,EAC/B,MAAM,uCAAuC,CAAC;AAE/C,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAE,+BAA+B,EAAE,8BAA8B,EAAE,CAAC;AAC3E,OAAO,EACL,kBAAkB,EAClB,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,GACd,CAAC;AACF,YAAY,EAAE,YAAY,EAAE,CAAC;AAE7B,MAAM,MAAM,oBAAoB,GAC5B;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACnC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,KAAK,CACV;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC9B;QAAE,IAAI,EAAE,OAAO,GAAG,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAChF,CAAC;CACH,GACC;IACA,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,KAAK,CACV;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC9B;QACA,IAAI,EAAE,WAAW,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,OAAO,CAAC;QACf,gBAAgB,CAAC,EAAE,OAAO,CAAC;KAC5B,GACC;QAOA,IAAI,EAAE,WAAW,CAAC;QAClB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;KACvB,CACF,CAAC;CACH,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,aAAa,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,OAAO,CAAA;SAAE,CAAC;KAC1C,CAAC,CAAC;CACJ,CAAC;AACJ,KAAK,qBAAqB,GACtB;IACA,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB,GACC;IACA,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,GAAG,MAAM,IAAI,MAAM,EAAE,CAAC;IAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAC/B,CAAC;AAkLJ,MAAM,MAAM,2BAA2B,GACnC;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACnC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EACH,MAAM,GACN,KAAK,CACH;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAC9B;QAAE,IAAI,EAAE,WAAW,CAAC;QAAC,SAAS,EAAE;YAAE,GAAG,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CACpD,CAAC;CACL,GACC;IACA,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,UAAU,CAAC,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,UAAU,CAAC;QACjB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;SACnB,CAAC;KACH,CAAC,CAAC;CACJ,GACC;IACA,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAMJ,MAAM,MAAM,2BAA2B,GAAG;IACxC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,2BAA2B,EAAE,CAAC;IACxC,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,cAAc,CAAC,EAAE;QACf,aAAa,CAAC,EAAE,OAAO,CAAC;KACzB,CAAC;IACF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,UAAU,CAAC;QACjB,QAAQ,EAAE;YACR,IAAI,EAAE,MAAM,CAAC;YACb,UAAU,EAAE,OAAO,CAAC;YACpB,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;KACH,CAAC,CAAC;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB,CAAC;AACF;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,YAAY,CAAC;CACxB,CAAC;AAEF;;;;GAIG;AACH,KAAK,gBAAgB,GAAG;IACtB,IAAI,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAC;IACrC,KAAK,IAAI,eAAe,EAAE,CAAC;CAC5B,CAAC;AAEF,wBAAgB,sBAAsB,IAAI,gBAAgB,CAUzD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMzD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GAAG,MAAM,CAQnF;AA0BD,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,oBAAoB,EAAE,GAC7B,2BAA2B,EAAE,CA0D/B;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,qBAAqB,EAAE,GAAG,SAAS,GACzC,2BAA2B,CAAC,OAAO,CAAC,GAAG,SAAS,CAmBlD;AAED,wBAAgB,mBAAmB,CACjC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EACpD,GAAG,aAAa,EAAE,MAAM,EAAE,GACzB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAezB;AAED,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,OAAO,GAAG,OAAO,CAOnE"}
package/esm/src/provider/runtime-loader.js CHANGED
@@ -32,6 +32,24 @@ export function readTextParts(parts) {
32
32
  }
33
33
  return text;
34
34
  }
35
+ function toOpenAICompatibleUserContent(parts) {
36
+ if (!parts.some((part) => part.type !== "text" && part.mediaType.startsWith("image/"))) {
37
+ return readTextParts(parts);
38
+ }
39
+ const content = [];
40
+ for (const part of parts) {
41
+ if (part.type === "text") {
42
+ if (part.text.length > 0) {
43
+ content.push({ type: "text", text: part.text });
44
+ }
45
+ continue;
46
+ }
47
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
48
+ content.push({ type: "image_url", image_url: { url: part.url } });
49
+ }
50
+ }
51
+ return content.length > 0 ? content : readTextParts(parts);
52
+ }
35
53
  export function toOpenAICompatibleMessages(prompt) {
36
54
  const messages = [];
37
55
  for (const message of prompt) {
@@ -40,7 +58,7 @@ export function toOpenAICompatibleMessages(prompt) {
40
58
  messages.push({ role: "system", content: message.content });
41
59
  break;
42
60
  case "user":
43
- messages.push({ role: "user", content: readTextParts(message.content) });
61
+ messages.push({ role: "user", content: toOpenAICompatibleUserContent(message.content) });
44
62
  break;
45
63
  case "assistant": {
46
64
  let text = "";
package/esm/src/runtime/runtime-bridge.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"runtime-bridge.d.ts","sourceRoot":"","sources":["../../../src/src/runtime/runtime-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,2DAA2D,CAAC;AAC9G,OAAO,KAAK,EACV,yBAAyB,EACzB,mBAAmB,EACnB,6BAA6B,EAC7B,cAAc,EACf,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,gBAAgB,EAChB,YAAY,EAEb,MAAM,sBAAsB,CAAC;AAE9B,KAAK,mBAAmB,GAAG;IACzB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,2BAA2B,CAAC,EAAE,6BAA6B,CAAC;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,2BAA2B,CAAC,EAAE,6BAA6B,CAAC;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,YAAY,GAAG;IAClB,KAAK,EAAE,gBAAgB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,gBAAgB,GAAG;IACtB,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAsaF,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAuBjG;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,mBAAmB,CAsC1E;AAED,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY;;;;;;;;GAW1C;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB;;;;;;;GAUlD;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBjE"}
1
+ {"version":3,"file":"runtime-bridge.d.ts","sourceRoot":"","sources":["../../../src/src/runtime/runtime-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,2DAA2D,CAAC;AAC9G,OAAO,KAAK,EACV,yBAAyB,EACzB,mBAAmB,EACnB,6BAA6B,EAC7B,cAAc,EACf,MAAM,wCAAwC,CAAC;AAChD,OAAO,KAAK,EACV,gBAAgB,EAChB,YAAY,EAEb,MAAM,sBAAsB,CAAC;AAE9B,KAAK,mBAAmB,GAAG;IACzB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,2BAA2B,CAAC,EAAE,6BAA6B,CAAC;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,4BAA4B,EAAE,CAAC;IACzC,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,2BAA2B,CAAC,EAAE,6BAA6B,CAAC;IAC5D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,YAAY,GAAG;IAClB,KAAK,EAAE,gBAAgB,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AAEF,KAAK,gBAAgB,GAAG;IACtB,KAAK,EAAE,gBAAgB,CAAC;IACxB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B,CAAC;AA8aF,wBAAgB,YAAY,CAAC,OAAO,EAAE,mBAAmB,GAAG,WAAW,CAAC,yBAAyB,CAAC,CAuBjG;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,iBAAiB,GAAG,mBAAmB,CAsC1E;AAED,wBAAgB,KAAK,CAAC,OAAO,EAAE,YAAY;;;;;;;;GAW1C;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB;;;;;;;GAUlD;AAED,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAsBjE"}
package/esm/src/runtime/runtime-bridge.js CHANGED
@@ -29,7 +29,9 @@ function toRuntimePrompt(system, messages) {
29
29
  case "user":
30
30
  prompt.push({
31
31
  role: "user",
32
- content: [{ type: "text", text: message.content }],
32
+ content: typeof message.content === "string"
33
+ ? [{ type: "text", text: message.content }]
34
+ : message.content,
33
35
  });
34
36
  break;
35
37
  case "assistant":
package/esm/src/utils/version-constant.d.ts CHANGED
@@ -1,2 +1,2 @@
1
- export declare const VERSION = "0.1.438";
1
+ export declare const VERSION = "0.1.439";
2
2
  //# sourceMappingURL=version-constant.d.ts.map
package/esm/src/utils/version-constant.js CHANGED
@@ -1,3 +1,3 @@
1
1
  // Keep in sync with deno.json version.
2
2
  // scripts/release.ts updates this constant during releases.
3
- export const VERSION = "0.1.438";
3
+ export const VERSION = "0.1.439";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "veryfront",
3
- "version": "0.1.438",
3
+ "version": "0.1.439",
4
4
  "description": "The simplest way to build AI-powered apps",
5
5
  "keywords": [
6
6
  "react",
package/src/deno.js CHANGED
@@ -1,6 +1,6 @@
1
1
  export default {
2
2
  "name": "veryfront",
3
- "version": "0.1.438",
3
+ "version": "0.1.439",
4
4
  "license": "Apache-2.0",
5
5
  "nodeModulesDir": "auto",
6
6
  "workspace": [
package/src/extensions/ext-anthropic/src/anthropic-provider.ts CHANGED
@@ -26,7 +26,6 @@ import {
26
26
  ProviderRequestError,
27
27
  readProviderOptions,
28
28
  readRecord,
29
- readTextParts,
30
29
  requestJson,
31
30
  requestStream,
32
31
  stringifyJsonValue,
@@ -283,6 +282,33 @@ function pushAnthropicUserContent(
283
282
  });
284
283
  }
285
284
 
285
+ function toAnthropicUserContent(
286
+ parts: Extract<RuntimePromptMessage, { role: "user" }>["content"],
287
+ ): Array<Record<string, unknown>> {
288
+ const content: Array<Record<string, unknown>> = [];
289
+
290
+ for (const part of parts) {
291
+ if (part.type === "text") {
292
+ if (part.text.length > 0) {
293
+ content.push({ type: "text", text: part.text });
294
+ }
295
+ continue;
296
+ }
297
+
298
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
299
+ content.push({
300
+ type: "image",
301
+ source: {
302
+ type: "url",
303
+ url: part.url,
304
+ },
305
+ });
306
+ }
307
+ }
308
+
309
+ return content;
310
+ }
311
+
286
312
  /**
287
313
  * Resolves a {@link ProviderCacheTtl} into Anthropic's `cache_control` shape.
288
314
  *
@@ -320,10 +346,7 @@ function toAnthropicMessages(
320
346
  }
321
347
  break;
322
348
  case "user":
323
- pushAnthropicUserContent(messages, [{
324
- type: "text",
325
- text: readTextParts(message.content),
326
- }]);
349
+ pushAnthropicUserContent(messages, toAnthropicUserContent(message.content));
327
350
  break;
328
351
  case "assistant":
329
352
  messages.push({
package/src/extensions/ext-google/src/google-provider.ts CHANGED
@@ -27,7 +27,6 @@ import {
27
27
  ProviderRequestError,
28
28
  readProviderOptions,
29
29
  readRecord,
30
- readTextParts,
31
30
  requestJson,
32
31
  requestStream,
33
32
  stringifyJsonValue,
@@ -249,7 +248,7 @@ function toGoogleContents(
249
248
  case "user":
250
249
  contents.push({
251
250
  role: "user",
252
- parts: [{ text: readTextParts(message.content) }],
251
+ parts: toGoogleUserParts(message.content),
253
252
  });
254
253
  break;
255
254
  case "assistant": {
@@ -300,6 +299,32 @@ function toGoogleContents(
300
299
  };
301
300
  }
302
301
 
302
+ function toGoogleUserParts(
303
+ parts: Extract<RuntimePromptMessage, { role: "user" }>["content"],
304
+ ): Array<Record<string, unknown>> {
305
+ const content: Array<Record<string, unknown>> = [];
306
+
307
+ for (const part of parts) {
308
+ if (part.type === "text") {
309
+ if (part.text.length > 0) {
310
+ content.push({ text: part.text });
311
+ }
312
+ continue;
313
+ }
314
+
315
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
316
+ content.push({
317
+ fileData: {
318
+ mimeType: part.mediaType,
319
+ fileUri: part.url,
320
+ },
321
+ });
322
+ }
323
+ }
324
+
325
+ return content;
326
+ }
327
+
303
328
  function toGoogleTools(
304
329
  tools: RuntimeToolDefinition[] | undefined,
305
330
  ): Array<Record<string, unknown>> | undefined {
package/src/extensions/ext-openai/src/openai-provider.ts CHANGED
@@ -27,7 +27,6 @@ import {
27
27
  ProviderRequestError,
28
28
  readProviderOptions,
29
29
  readRecord,
30
- readTextParts,
31
30
  requestJson,
32
31
  requestStream,
33
32
  stringifyJsonValue,
@@ -751,7 +750,7 @@ function toOpenAIResponsesInput(
751
750
  case "user":
752
751
  input.push({
753
752
  role: "user",
754
- content: [{ type: "input_text", text: readTextParts(message.content) }],
753
+ content: toOpenAIResponsesUserContent(message.content),
755
754
  });
756
755
  break;
757
756
  case "assistant": {
@@ -816,6 +815,34 @@ function toOpenAIResponsesInput(
816
815
  };
817
816
  }
818
817
 
818
+ function toOpenAIResponsesUserContent(
819
+ parts: Extract<RuntimePromptMessage, { role: "user" }>["content"],
820
+ ): Array<Record<string, unknown>> {
821
+ const content: Array<Record<string, unknown>> = [];
822
+
823
+ for (const part of parts) {
824
+ if (part.type === "text") {
825
+ if (part.text.length > 0) {
826
+ content.push({ type: "input_text", text: part.text });
827
+ }
828
+ continue;
829
+ }
830
+
831
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
832
+ content.push({ type: "input_image", image_url: part.url, detail: "auto" });
833
+ continue;
834
+ }
835
+
836
+ content.push({
837
+ type: "input_file",
838
+ file_url: part.url,
839
+ ...(part.filename ? { filename: part.filename } : {}),
840
+ });
841
+ }
842
+
843
+ return content;
844
+ }
845
+
819
846
  /**
820
847
  * Tools on the Responses API differ from Chat Completions: instead of
821
848
  * `{ type: "function", function: { name, parameters } }` the function
package/src/src/agent/runtime/text-generation-runtime-message-converter.ts CHANGED
@@ -9,6 +9,7 @@
9
9
 
10
10
  import type {
11
11
  TextGenerationRuntimeAssistantMessage,
12
+ TextGenerationRuntimeFilePart,
12
13
  TextGenerationRuntimeMessage,
13
14
  TextGenerationRuntimeTextPart,
14
15
  TextGenerationRuntimeToolCallPart,
@@ -75,6 +76,26 @@ function getUserTextWithAttachmentContext(parts: Message["parts"]): string {
75
76
  : appendReadableAttachmentContext(text, buildAttachmentContextFromParts(parts));
76
77
  }
77
78
 
79
+ function getUserFileParts(parts: Message["parts"]): TextGenerationRuntimeFilePart[] {
80
+ return parts.flatMap((part) => {
81
+ const type = getStringPartField(part, "type");
82
+ if (type !== "file" && type !== "image") return [];
83
+
84
+ const mediaType = getStringPartField(part, "mediaType");
85
+ const url = getStringPartField(part, "url");
86
+ if (!mediaType || !url || url.startsWith("data:")) return [];
87
+
88
+ return [{
89
+ type,
90
+ mediaType,
91
+ url,
92
+ ...(getStringPartField(part, "filename")
93
+ ? { filename: getStringPartField(part, "filename") }
94
+ : {}),
95
+ }];
96
+ });
97
+ }
98
+
78
99
  /**
79
100
  * Convert a veryfront Message to the current text-generation runtime message format.
80
101
  */
@@ -86,8 +107,26 @@ export function convertToTextGenerationRuntimeMessage(msg: Message): TextGenerat
86
107
  }
87
108
 
88
109
  case "user": {
89
- const text = getUserTextWithAttachmentContext(msg.parts);
90
- return { role: "user", content: text };
110
+ const fileParts = getUserFileParts(msg.parts);
111
+ if (fileParts.length === 0) {
112
+ const text = getUserTextWithAttachmentContext(msg.parts);
113
+ return { role: "user", content: text };
114
+ }
115
+
116
+ const text = getTextFromParts(msg.parts);
117
+ const attachmentContext = text.includes("<uploaded_files>")
118
+ ? ""
119
+ : buildAttachmentContextFromParts(msg.parts);
120
+ return {
121
+ role: "user",
122
+ content: [
123
+ ...(text.length > 0 ? [{ type: "text" as const, text }] : []),
124
+ ...fileParts,
125
+ ...(attachmentContext.length > 0
126
+ ? [{ type: "text" as const, text: attachmentContext.trimStart() }]
127
+ : []),
128
+ ],
129
+ };
91
130
  }
92
131
 
93
132
  case "assistant": {
package/src/src/agent/runtime/text-generation-runtime-message-types.ts CHANGED
@@ -11,6 +11,13 @@ export interface TextGenerationRuntimeTextPart {
11
11
  text: string;
12
12
  }
13
13
 
14
+ export interface TextGenerationRuntimeFilePart {
15
+ type: "file" | "image";
16
+ mediaType: string;
17
+ url: string;
18
+ filename?: string;
19
+ }
20
+
14
21
  export interface TextGenerationRuntimeToolCallPart {
15
22
  type: "tool-call";
16
23
  toolCallId: string;
@@ -35,7 +42,7 @@ export interface TextGenerationRuntimeSystemMessage {
35
42
 
36
43
  export interface TextGenerationRuntimeUserMessage {
37
44
  role: "user";
38
- content: string;
45
+ content: string | Array<TextGenerationRuntimeTextPart | TextGenerationRuntimeFilePart>;
39
46
  }
40
47
 
41
48
  export interface TextGenerationRuntimeAssistantMessage {
package/src/src/agent/runtime-message-file-url-refresh.ts CHANGED
@@ -1,4 +1,4 @@
1
- import type { ChatFileUiPart, ChatUiMessage, FileUIPartWithUpload } from "../chat/types.js";
1
+ import type { ChatUiMessage, FileUIPartWithUpload } from "../chat/types.js";
2
2
 
3
3
  export type RuntimeFileUrlResolverInput = {
4
4
  uploadId: string;
@@ -18,25 +18,30 @@ export async function resolveRuntimeMessageFileUrls(
18
18
 
19
19
  return Promise.all(
20
20
  messages.map(async (message) => {
21
- if (!message.parts.some((part) => part.type === "file" && part.uploadId)) {
21
+ if (!message.parts.some((part) => getUploadId(part))) {
22
22
  return message;
23
23
  }
24
24
 
25
25
  const parts = await Promise.all(
26
26
  message.parts.map(async (part) => {
27
- if (!isFilePartWithUpload(part)) return part;
27
+ const uploadId = getUploadId(part);
28
+ if (!uploadId) return part;
28
29
 
29
- let urlPromise = urlByUploadId.get(part.uploadId);
30
+ let urlPromise = urlByUploadId.get(uploadId);
30
31
  if (!urlPromise) {
31
- urlPromise = resolveFileUrl({ uploadId: part.uploadId, part, message });
32
- urlByUploadId.set(part.uploadId, urlPromise);
32
+ urlPromise = resolveFileUrl({
33
+ uploadId,
34
+ part: toResolverPart(part, uploadId),
35
+ message,
36
+ });
37
+ urlByUploadId.set(uploadId, urlPromise);
33
38
  }
34
39
 
35
40
  const signedUrl = await urlPromise;
36
- if (!signedUrl || signedUrl === part.url) return part;
41
+ if (!signedUrl) return normalizeUploadedFilePart(part, uploadId);
37
42
 
38
43
  return {
39
- ...part,
44
+ ...normalizeUploadedFilePart(part, uploadId),
40
45
  url: signedUrl,
41
46
  };
42
47
  }),
@@ -47,8 +52,69 @@ export async function resolveRuntimeMessageFileUrls(
47
52
  );
48
53
  }
49
54
 
50
- function isFilePartWithUpload(part: ChatUiMessage["parts"][number]): part is ChatFileUiPart & {
51
- uploadId: string;
52
- } {
53
- return part.type === "file" && typeof part.uploadId === "string" && part.uploadId.length > 0;
55
+ function isRecord(value: unknown): value is Record<string, unknown> {
56
+ return typeof value === "object" && value !== null && !Array.isArray(value);
57
+ }
58
+
59
+ function getStringField(part: unknown, key: string): string | undefined {
60
+ if (!isRecord(part)) return undefined;
61
+
62
+ const value = part[key];
63
+ return typeof value === "string" && value.length > 0 ? value : undefined;
64
+ }
65
+
66
+ function getUploadId(part: unknown): string | undefined {
67
+ if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
68
+ return undefined;
69
+ }
70
+
71
+ return getStringField(part, "uploadId") ?? getStringField(part, "upload_id");
72
+ }
73
+
74
+ function getMediaType(part: unknown): string | undefined {
75
+ return getStringField(part, "mediaType") ?? getStringField(part, "media_type");
76
+ }
77
+
78
+ function normalizeUploadedFilePart(
79
+ part: ChatUiMessage["parts"][number],
80
+ uploadId: string,
81
+ ): ChatUiMessage["parts"][number] {
82
+ if (!isRecord(part)) return part;
83
+
84
+ const partRecord: Record<string, unknown> = part;
85
+ const partType = partRecord.type;
86
+ if (partType !== "file" && partType !== "image") return part;
87
+
88
+ const mediaType = getMediaType(part);
89
+ const url = getStringField(part, "url");
90
+ if (!mediaType || !url) return part;
91
+
92
+ const filename = getStringField(part, "filename");
93
+ const uploadPath = getStringField(part, "uploadPath") ?? getStringField(part, "upload_path");
94
+
95
+ return {
96
+ type: partType === "image" ? "image" : "file",
97
+ mediaType,
98
+ url,
99
+ ...(filename ? { filename } : {}),
100
+ uploadId,
101
+ ...(uploadPath ? { uploadPath } : {}),
102
+ } as ChatUiMessage["parts"][number];
103
+ }
104
+
105
+ function toResolverPart(
106
+ part: ChatUiMessage["parts"][number],
107
+ uploadId: string,
108
+ ): FileUIPartWithUpload {
109
+ const normalized = normalizeUploadedFilePart(part, uploadId);
110
+ if (isRecord(normalized) && normalized.type === "file") {
111
+ return normalized as FileUIPartWithUpload;
112
+ }
113
+
114
+ return {
115
+ type: "file",
116
+ mediaType: getMediaType(part) ?? "application/octet-stream",
117
+ url: getStringField(part, "url") ?? "",
118
+ uploadId,
119
+ };
54
120
  }
package/src/src/chat/conversation.ts CHANGED
@@ -515,7 +515,7 @@ function toJsonValue(value: unknown): JsonValue {
515
515
  }
516
516
 
517
517
  function getFilePart(part: unknown): {
518
- type: "file";
518
+ type: "file" | "image";
519
519
  mediaType: string;
520
520
  data: string;
521
521
  url: string;
@@ -523,22 +523,25 @@ function getFilePart(part: unknown): {
523
523
  uploadId?: string;
524
524
  uploadPath?: string;
525
525
  } | null {
526
- if (!isRecord(part) || part.type !== "file") {
526
+ if (!isRecord(part) || (part.type !== "file" && part.type !== "image")) {
527
527
  return null;
528
528
  }
529
529
 
530
- const mediaType = getNonEmptyStringField(part, "mediaType");
530
+ const mediaType = getNonEmptyStringField(part, "mediaType") ??
531
+ getNonEmptyStringField(part, "media_type");
531
532
  const data = getNonEmptyStringField(part, "url");
532
533
  if (!mediaType || !data) {
533
534
  return null;
534
535
  }
535
536
 
536
537
  const filename = getNonEmptyStringField(part, "filename");
537
- const uploadId = getNonEmptyStringField(part, "uploadId");
538
- const uploadPath = getNonEmptyStringField(part, "uploadPath");
538
+ const uploadId = getNonEmptyStringField(part, "uploadId") ??
539
+ getNonEmptyStringField(part, "upload_id");
540
+ const uploadPath = getNonEmptyStringField(part, "uploadPath") ??
541
+ getNonEmptyStringField(part, "upload_path");
539
542
 
540
543
  return {
541
- type: "file",
544
+ type: part.type === "image" ? "image" : "file",
542
545
  mediaType,
543
546
  data,
544
547
  url: data,
@@ -721,10 +724,13 @@ function convertSystemMessage(message: ChatUiMessage): ProviderModelMessage[] {
721
724
  function convertUserMessage(message: ChatUiMessage): ProviderModelMessage[] {
722
725
  const content: Array<
723
726
  { type: "text"; text: string } | {
724
- type: "file";
727
+ type: "file" | "image";
725
728
  mediaType: string;
726
729
  data: string;
730
+ url: string;
727
731
  filename?: string;
732
+ uploadId?: string;
733
+ uploadPath?: string;
728
734
  }
729
735
  > = [];
730
736
 
@@ -757,7 +763,7 @@ function convertAssistantMessage(message: ChatUiMessage): ProviderModelMessage[]
757
763
  const assistantContent: Array<
758
764
  | { type: "text"; text: string }
759
765
  | { type: "reasoning"; text: string }
760
- | { type: "file"; mediaType: string; data: string; filename?: string }
766
+ | { type: "file" | "image"; mediaType: string; data: string; filename?: string }
761
767
  | { type: "tool-call"; toolCallId: string; toolName: string; input: Record<string, unknown> }
762
768
  > = [];
763
769
  const deferredAssistantContent: typeof assistantContent = [];
@@ -806,7 +812,7 @@ function convertAssistantMessage(message: ChatUiMessage): ProviderModelMessage[]
806
812
  part:
807
813
  | { type: "text"; text: string }
808
814
  | { type: "reasoning"; text: string }
809
- | { type: "file"; mediaType: string; data: string; filename?: string }
815
+ | { type: "file" | "image"; mediaType: string; data: string; filename?: string }
810
816
  | { type: "tool-call"; toolCallId: string; toolName: string; input: Record<string, unknown> },
811
817
  ) => {
812
818
  if (toolResults.length > 0) {
package/src/src/provider/runtime-loader.ts CHANGED
@@ -34,7 +34,13 @@ export type { RuntimeUsage };
34
34
 
35
35
  export type RuntimePromptMessage =
36
36
  | { role: "system"; content: string }
37
- | { role: "user"; content: Array<{ type: "text"; text: string }> }
37
+ | {
38
+ role: "user";
39
+ content: Array<
40
+ | { type: "text"; text: string }
41
+ | { type: "image" | "file"; mediaType: string; url: string; filename?: string }
42
+ >;
43
+ }
38
44
  | {
39
45
  role: "assistant";
40
46
  content: Array<
@@ -261,7 +267,15 @@ type OpenAICompatibleLanguageOptions = {
261
267
  };
262
268
  export type OpenAICompatibleChatMessage =
263
269
  | { role: "system"; content: string }
264
- | { role: "user"; content: string }
270
+ | {
271
+ role: "user";
272
+ content:
273
+ | string
274
+ | Array<
275
+ | { type: "text"; text: string }
276
+ | { type: "image_url"; image_url: { url: string } }
277
+ >;
278
+ }
265
279
  | {
266
280
  role: "assistant";
267
281
  content: string | null;
@@ -279,6 +293,11 @@ export type OpenAICompatibleChatMessage =
279
293
  tool_call_id: string;
280
294
  content: string;
281
295
  };
296
+ type RuntimePromptUserContent = Extract<RuntimePromptMessage, { role: "user" }>["content"];
297
+ type OpenAICompatibleUserContent = Extract<
298
+ OpenAICompatibleChatMessage,
299
+ { role: "user" }
300
+ >["content"];
282
301
  export type OpenAICompatibleChatRequest = {
283
302
  model: string;
284
303
  messages: OpenAICompatibleChatMessage[];
@@ -359,6 +378,30 @@ export function readTextParts(parts: Array<{ type: string; text?: string }>): st
359
378
  return text;
360
379
  }
361
380
 
381
+ function toOpenAICompatibleUserContent(
382
+ parts: RuntimePromptUserContent,
383
+ ): OpenAICompatibleUserContent {
384
+ if (!parts.some((part) => part.type !== "text" && part.mediaType.startsWith("image/"))) {
385
+ return readTextParts(parts);
386
+ }
387
+
388
+ const content: Exclude<OpenAICompatibleUserContent, string> = [];
389
+
390
+ for (const part of parts) {
391
+ if (part.type === "text") {
392
+ if (part.text.length > 0) {
393
+ content.push({ type: "text", text: part.text });
394
+ }
395
+ continue;
396
+ }
397
+ if (part.type === "image" || part.mediaType.startsWith("image/")) {
398
+ content.push({ type: "image_url", image_url: { url: part.url } });
399
+ }
400
+ }
401
+
402
+ return content.length > 0 ? content : readTextParts(parts);
403
+ }
404
+
362
405
  export function toOpenAICompatibleMessages(
363
406
  prompt: RuntimePromptMessage[],
364
407
  ): OpenAICompatibleChatMessage[] {
@@ -370,7 +413,7 @@ export function toOpenAICompatibleMessages(
370
413
  messages.push({ role: "system", content: message.content });
371
414
  break;
372
415
  case "user":
373
- messages.push({ role: "user", content: readTextParts(message.content) });
416
+ messages.push({ role: "user", content: toOpenAICompatibleUserContent(message.content) });
374
417
  break;
375
418
  case "assistant": {
376
419
  let text = "";
package/src/src/runtime/runtime-bridge.ts CHANGED
@@ -73,7 +73,13 @@ type EmbedManyOptions = {
73
73
 
74
74
  type RuntimePromptMessage =
75
75
  | { role: "system"; content: string }
76
- | { role: "user"; content: Array<{ type: "text"; text: string }> }
76
+ | {
77
+ role: "user";
78
+ content: Array<
79
+ | { type: "text"; text: string }
80
+ | { type: "image" | "file"; mediaType: string; url: string; filename?: string }
81
+ >;
82
+ }
77
83
  | {
78
84
  role: "assistant";
79
85
  content: Array<
@@ -181,7 +187,9 @@ function toRuntimePrompt(
181
187
  case "user":
182
188
  prompt.push({
183
189
  role: "user",
184
- content: [{ type: "text", text: message.content }],
190
+ content: typeof message.content === "string"
191
+ ? [{ type: "text", text: message.content }]
192
+ : message.content,
185
193
  });
186
194
  break;
187
195
  case "assistant":
package/src/src/utils/version-constant.ts CHANGED
@@ -1,3 +1,3 @@
1
1
  // Keep in sync with deno.json version.
2
2
  // scripts/release.ts updates this constant during releases.
3
- export const VERSION = "0.1.438";
3
+ export const VERSION = "0.1.439";