synthos 0.10.1 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (311) hide show
  1. package/README.md +5 -5
  2. package/default-pages/elevenlabs_effects_studio/chat-history.json +1 -0
  3. package/default-pages/elevenlabs_effects_studio/page.html +1345 -1363
  4. package/default-pages/elevenlabs_effects_studio/page.json +13 -11
  5. package/default-pages/elevenlabs_voice_studio/chat-history.json +1 -0
  6. package/default-pages/elevenlabs_voice_studio/page.html +782 -801
  7. package/default-pages/elevenlabs_voice_studio/page.json +13 -11
  8. package/default-pages/json_tools/chat-history.json +1 -0
  9. package/default-pages/json_tools/page.html +70 -90
  10. package/default-pages/json_tools/page.json +12 -10
  11. package/default-pages/my_notes/chat-history.json +1 -0
  12. package/default-pages/my_notes/page.html +115 -131
  13. package/default-pages/my_notes/page.json +14 -12
  14. package/default-pages/neon_asteroids/chat-history.json +1 -0
  15. package/default-pages/neon_asteroids/page.html +1777 -1803
  16. package/default-pages/neon_asteroids/page.json +14 -12
  17. package/default-pages/oregon_trail/chat-history.json +1 -0
  18. package/default-pages/oregon_trail/page.html +290 -307
  19. package/default-pages/oregon_trail/page.json +14 -12
  20. package/default-pages/solar_explorer/chat-history.json +1 -0
  21. package/default-pages/solar_explorer/page.html +1929 -1951
  22. package/default-pages/solar_explorer/page.json +14 -12
  23. package/default-pages/solar_tutorial/chat-history.json +1 -0
  24. package/default-pages/solar_tutorial/page.html +464 -478
  25. package/default-pages/solar_tutorial/page.json +12 -10
  26. package/default-pages/us_map/chat-history.json +1 -0
  27. package/default-pages/us_map/page.html +170 -193
  28. package/default-pages/us_map/page.json +14 -12
  29. package/default-pages/us_map/page.light.png +0 -0
  30. package/default-pages/us_map_1850/chat-history.json +1 -0
  31. package/default-pages/us_map_1850/page.html +302 -326
  32. package/default-pages/us_map_1850/page.json +14 -12
  33. package/default-pages/western_cities_1850/chat-history.json +1 -0
  34. package/default-pages/western_cities_1850/page.html +503 -527
  35. package/default-pages/western_cities_1850/page.json +14 -12
  36. package/default-themes/aurora-dawn.v3.css +15 -14
  37. package/default-themes/aurora-dusk.v3.css +26 -26
  38. package/default-themes/cosmos-dawn.v3.css +15 -14
  39. package/default-themes/cosmos-dusk.v3.css +26 -26
  40. package/default-themes/elemental-dawn.v3.css +200 -0
  41. package/default-themes/nebula-dawn.v3.css +15 -14
  42. package/default-themes/nebula-dusk.v3.css +24 -24
  43. package/default-themes/solar-flare-dawn.v3.css +15 -14
  44. package/default-themes/solar-flare-dusk.v3.css +26 -26
  45. package/dist/builders/anthropic.d.ts +26 -2
  46. package/dist/builders/anthropic.d.ts.map +1 -1
  47. package/dist/builders/anthropic.js +132 -31
  48. package/dist/builders/anthropic.js.map +1 -1
  49. package/dist/builders/claudecode.d.ts +13 -0
  50. package/dist/builders/claudecode.d.ts.map +1 -0
  51. package/dist/builders/claudecode.js +253 -0
  52. package/dist/builders/claudecode.js.map +1 -0
  53. package/dist/builders/index.d.ts +2 -1
  54. package/dist/builders/index.d.ts.map +1 -1
  55. package/dist/builders/index.js +8 -1
  56. package/dist/builders/index.js.map +1 -1
  57. package/dist/builders/openai.js +2 -1
  58. package/dist/builders/openai.js.map +1 -1
  59. package/dist/builders/types.d.ts +31 -7
  60. package/dist/builders/types.d.ts.map +1 -1
  61. package/dist/builders/types.js +60 -28
  62. package/dist/builders/types.js.map +1 -1
  63. package/dist/connectors/types.d.ts +8 -0
  64. package/dist/connectors/types.d.ts.map +1 -1
  65. package/dist/init.d.ts.map +1 -1
  66. package/dist/init.js +13 -6
  67. package/dist/init.js.map +1 -1
  68. package/dist/migrations.d.ts.map +1 -1
  69. package/dist/migrations.js +161 -14
  70. package/dist/migrations.js.map +1 -1
  71. package/dist/models/anthropic.d.ts +1 -0
  72. package/dist/models/anthropic.d.ts.map +1 -1
  73. package/dist/models/anthropic.js +129 -29
  74. package/dist/models/anthropic.js.map +1 -1
  75. package/dist/models/chainOfThought.d.ts.map +1 -1
  76. package/dist/models/chainOfThought.js +32 -19
  77. package/dist/models/chainOfThought.js.map +1 -1
  78. package/dist/models/index.d.ts +2 -2
  79. package/dist/models/index.d.ts.map +1 -1
  80. package/dist/models/index.js +2 -1
  81. package/dist/models/index.js.map +1 -1
  82. package/dist/models/providers.d.ts +1 -0
  83. package/dist/models/providers.d.ts.map +1 -1
  84. package/dist/models/providers.js +12 -4
  85. package/dist/models/providers.js.map +1 -1
  86. package/dist/models/types.d.ts +15 -1
  87. package/dist/models/types.d.ts.map +1 -1
  88. package/dist/models/types.js.map +1 -1
  89. package/dist/pages.d.ts +57 -8
  90. package/dist/pages.d.ts.map +1 -1
  91. package/dist/pages.js +258 -45
  92. package/dist/pages.js.map +1 -1
  93. package/dist/service/createCompletePrompt.d.ts.map +1 -1
  94. package/dist/service/createCompletePrompt.js +5 -0
  95. package/dist/service/createCompletePrompt.js.map +1 -1
  96. package/dist/service/mediaCache.d.ts +36 -0
  97. package/dist/service/mediaCache.d.ts.map +1 -0
  98. package/dist/service/mediaCache.js +182 -0
  99. package/dist/service/mediaCache.js.map +1 -0
  100. package/dist/service/pageValidator.d.ts +25 -0
  101. package/dist/service/pageValidator.d.ts.map +1 -0
  102. package/dist/service/pageValidator.js +315 -0
  103. package/dist/service/pageValidator.js.map +1 -0
  104. package/dist/service/server.d.ts.map +1 -1
  105. package/dist/service/server.js +4 -0
  106. package/dist/service/server.js.map +1 -1
  107. package/dist/service/sharedTableSchema.d.ts +73 -0
  108. package/dist/service/sharedTableSchema.d.ts.map +1 -0
  109. package/dist/service/sharedTableSchema.js +206 -0
  110. package/dist/service/sharedTableSchema.js.map +1 -0
  111. package/dist/service/transformPage.d.ts +49 -11
  112. package/dist/service/transformPage.d.ts.map +1 -1
  113. package/dist/service/transformPage.js +354 -241
  114. package/dist/service/transformPage.js.map +1 -1
  115. package/dist/service/useApiRoutes.d.ts.map +1 -1
  116. package/dist/service/useApiRoutes.js +288 -34
  117. package/dist/service/useApiRoutes.js.map +1 -1
  118. package/dist/service/useConnectorRoutes.d.ts.map +1 -1
  119. package/dist/service/useConnectorRoutes.js +170 -32
  120. package/dist/service/useConnectorRoutes.js.map +1 -1
  121. package/dist/service/useDataRoutes.d.ts.map +1 -1
  122. package/dist/service/useDataRoutes.js +59 -2
  123. package/dist/service/useDataRoutes.js.map +1 -1
  124. package/dist/service/useExtractRoutes.d.ts +4 -0
  125. package/dist/service/useExtractRoutes.d.ts.map +1 -0
  126. package/dist/service/useExtractRoutes.js +304 -0
  127. package/dist/service/useExtractRoutes.js.map +1 -0
  128. package/dist/service/usePageRoutes.d.ts +17 -0
  129. package/dist/service/usePageRoutes.d.ts.map +1 -1
  130. package/dist/service/usePageRoutes.js +1385 -483
  131. package/dist/service/usePageRoutes.js.map +1 -1
  132. package/dist/service/useSharedDataRoutes.d.ts.map +1 -1
  133. package/dist/service/useSharedDataRoutes.js +54 -2
  134. package/dist/service/useSharedDataRoutes.js.map +1 -1
  135. package/dist/settings.d.ts +27 -0
  136. package/dist/settings.d.ts.map +1 -1
  137. package/dist/settings.js +40 -1
  138. package/dist/settings.js.map +1 -1
  139. package/dist/themes.d.ts +0 -5
  140. package/dist/themes.d.ts.map +1 -1
  141. package/dist/themes.js +3 -95
  142. package/dist/themes.js.map +1 -1
  143. package/migration-rules/v2-to-v3.md +277 -119
  144. package/package.json +5 -1
  145. package/{default-pages/application → required-pages/_shell}/page.html +56 -42
  146. package/required-pages/_shell/page.json +14 -0
  147. package/required-pages/_starters/page.html +534 -0
  148. package/required-pages/_starters/page.json +12 -0
  149. package/required-pages/builder/page.html +353 -43
  150. package/required-pages/builder/page.json +12 -10
  151. package/required-pages/pages/page.html +697 -924
  152. package/required-pages/pages/page.json +12 -10
  153. package/required-pages/settings/page.html +1879 -1753
  154. package/required-pages/settings/page.json +12 -10
  155. package/required-pages/synthos_apis/page.html +834 -845
  156. package/required-pages/synthos_apis/page.json +12 -10
  157. package/required-pages/synthos_scripts/page.html +74 -88
  158. package/required-pages/synthos_scripts/page.json +12 -10
  159. package/scripts/append-instructions.py +90 -0
  160. package/scripts/audit-instructions.py +76 -0
  161. package/scripts/cleanup-shell-markup.mjs +112 -0
  162. package/service-connectors/buffer/connector.json +46 -0
  163. package/service-connectors/canva/connector.json +67 -0
  164. package/service-connectors/elevenlabs/connector.json +1 -1
  165. package/src/builders/anthropic.ts +150 -25
  166. package/src/builders/claudecode.ts +310 -0
  167. package/src/builders/index.ts +7 -1
  168. package/src/builders/openai.ts +2 -1
  169. package/src/builders/types.ts +93 -32
  170. package/src/connectors/types.ts +8 -0
  171. package/src/init.ts +13 -7
  172. package/src/migrations.ts +187 -16
  173. package/src/models/anthropic.ts +140 -30
  174. package/src/models/chainOfThought.ts +33 -18
  175. package/src/models/index.ts +2 -2
  176. package/src/models/providers.ts +10 -1
  177. package/src/models/types.ts +21 -1
  178. package/src/pages.ts +271 -35
  179. package/src/service/createCompletePrompt.ts +6 -0
  180. package/src/service/mediaCache.ts +206 -0
  181. package/src/service/pageValidator.ts +337 -0
  182. package/src/service/server.ts +4 -0
  183. package/src/service/sharedTableSchema.ts +236 -0
  184. package/src/service/transformPage.ts +370 -260
  185. package/src/service/useApiRoutes.ts +282 -32
  186. package/src/service/useConnectorRoutes.ts +189 -34
  187. package/src/service/useDataRoutes.ts +198 -116
  188. package/src/service/useExtractRoutes.ts +331 -0
  189. package/src/service/usePageRoutes.ts +1411 -394
  190. package/src/service/useSharedDataRoutes.ts +184 -109
  191. package/src/settings.ts +65 -0
  192. package/src/themes.ts +78 -180
  193. package/starters/blank_starter/chat-history.json +1 -0
  194. package/starters/blank_starter/page.dark.png +0 -0
  195. package/starters/blank_starter/page.html +47 -0
  196. package/starters/blank_starter/page.json +13 -0
  197. package/starters/blank_starter/page.light.png +0 -0
  198. package/starters/calculator_starter/chat-history.json +1 -0
  199. package/starters/calculator_starter/page.dark.png +0 -0
  200. package/starters/calculator_starter/page.html +232 -0
  201. package/starters/calculator_starter/page.json +13 -0
  202. package/starters/calculator_starter/page.light.png +0 -0
  203. package/starters/calendar_starter/chat-history.json +1 -0
  204. package/starters/calendar_starter/page.dark.png +0 -0
  205. package/starters/calendar_starter/page.html +495 -0
  206. package/starters/calendar_starter/page.json +13 -0
  207. package/starters/calendar_starter/page.light.png +0 -0
  208. package/starters/chat_starter/chat-history.json +1 -0
  209. package/starters/chat_starter/page.dark.png +0 -0
  210. package/starters/chat_starter/page.html +351 -0
  211. package/starters/chat_starter/page.json +13 -0
  212. package/starters/chat_starter/page.light.png +0 -0
  213. package/starters/checklist_starter/chat-history.json +1 -0
  214. package/starters/checklist_starter/page.dark.png +0 -0
  215. package/starters/checklist_starter/page.html +437 -0
  216. package/starters/checklist_starter/page.json +13 -0
  217. package/starters/checklist_starter/page.light.png +0 -0
  218. package/starters/dashboard_starter/chat-history.json +1 -0
  219. package/starters/dashboard_starter/page.dark.png +0 -0
  220. package/starters/dashboard_starter/page.html +195 -0
  221. package/starters/dashboard_starter/page.json +13 -0
  222. package/starters/dashboard_starter/page.light.png +0 -0
  223. package/starters/form_starter/chat-history.json +1 -0
  224. package/starters/form_starter/page.dark.png +0 -0
  225. package/starters/form_starter/page.html +313 -0
  226. package/starters/form_starter/page.json +13 -0
  227. package/starters/form_starter/page.light.png +0 -0
  228. package/starters/gallery_starter/chat-history.json +1 -0
  229. package/starters/gallery_starter/page.dark.png +0 -0
  230. package/starters/gallery_starter/page.html +418 -0
  231. package/starters/gallery_starter/page.json +13 -0
  232. package/starters/gallery_starter/page.light.png +0 -0
  233. package/starters/generator_starter/chat-history.json +1 -0
  234. package/starters/generator_starter/page.dark.png +0 -0
  235. package/starters/generator_starter/page.html +261 -0
  236. package/starters/generator_starter/page.json +13 -0
  237. package/starters/generator_starter/page.light.png +0 -0
  238. package/starters/index.html +538 -0
  239. package/starters/kanban_starter/chat-history.json +1 -0
  240. package/starters/kanban_starter/page.dark.png +0 -0
  241. package/starters/kanban_starter/page.html +432 -0
  242. package/starters/kanban_starter/page.json +13 -0
  243. package/starters/kanban_starter/page.light.png +0 -0
  244. package/starters/presentation_builder/chat-history.json +1 -0
  245. package/starters/presentation_builder/page.dark.png +0 -0
  246. package/starters/presentation_builder/page.html +970 -0
  247. package/starters/presentation_builder/page.json +15 -0
  248. package/starters/presentation_builder/page.light.png +0 -0
  249. package/starters/presentation_builder/presentation_voice/voice_config.json +9 -0
  250. package/starters/pulse_starter/chat-history.json +1 -0
  251. package/starters/pulse_starter/page.dark.png +0 -0
  252. package/starters/pulse_starter/page.html +698 -0
  253. package/starters/pulse_starter/page.json +13 -0
  254. package/starters/pulse_starter/page.light.png +0 -0
  255. package/starters/quiz_starter/chat-history.json +1 -0
  256. package/starters/quiz_starter/page.dark.png +0 -0
  257. package/starters/quiz_starter/page.html +292 -0
  258. package/starters/quiz_starter/page.json +13 -0
  259. package/starters/quiz_starter/page.light.png +0 -0
  260. package/starters/reference_starter/chat-history.json +1 -0
  261. package/starters/reference_starter/page.dark.png +0 -0
  262. package/starters/reference_starter/page.html +250 -0
  263. package/starters/reference_starter/page.json +13 -0
  264. package/starters/reference_starter/page.light.png +0 -0
  265. package/starters/retro_game_starter/chat-history.json +1 -0
  266. package/starters/retro_game_starter/page.dark.png +0 -0
  267. package/{default-pages → starters}/retro_game_starter/page.html +1281 -1308
  268. package/starters/retro_game_starter/page.json +15 -0
  269. package/starters/retro_game_starter/page.light.png +0 -0
  270. package/starters/roster_starter/chat-history.json +1 -0
  271. package/starters/roster_starter/page.dark.png +0 -0
  272. package/starters/roster_starter/page.html +600 -0
  273. package/starters/roster_starter/page.json +13 -0
  274. package/starters/roster_starter/page.light.png +0 -0
  275. package/starters/server.js +182 -0
  276. package/starters/start.cmd +1 -0
  277. package/starters/timeline_starter/chat-history.json +1 -0
  278. package/starters/timeline_starter/page.dark.png +0 -0
  279. package/starters/timeline_starter/page.html +446 -0
  280. package/starters/timeline_starter/page.json +13 -0
  281. package/starters/timeline_starter/page.light.png +0 -0
  282. package/starters/tutorial_starter/chat-history.json +1 -0
  283. package/starters/tutorial_starter/page.dark.png +0 -0
  284. package/starters/tutorial_starter/page.html +283 -0
  285. package/starters/tutorial_starter/page.json +13 -0
  286. package/starters/tutorial_starter/page.light.png +0 -0
  287. package/static-files/agent.v3.js +122 -0
  288. package/static-files/connector.v3.js +48 -0
  289. package/static-files/extract.v3.js +188 -0
  290. package/static-files/helpers.v3.js +50 -6
  291. package/static-files/page-bridge.js +114 -0
  292. package/static-files/page.v3.js +1292 -1290
  293. package/static-files/script.v3.js +32 -0
  294. package/static-files/server.v3.js +89 -0
  295. package/static-files/shell-bridge.v3.js +174 -0
  296. package/static-files/shell-modals.v3.js +521 -0
  297. package/static-files/{shell.css → shell.v3.css} +271 -22
  298. package/static-files/shell.v3.js +1865 -0
  299. package/static-files/storage.v3.js +176 -0
  300. package/tests/anthropic.spec.ts +42 -7
  301. package/tests/builders.spec.ts +70 -2
  302. package/tests/pageValidator.spec.ts +548 -0
  303. package/tests/profiles.spec.ts +122 -0
  304. package/tests/sharedTableSchema.spec.ts +242 -0
  305. package/tests/transformPage.spec.ts +62 -81
  306. package/default-pages/application/page.json +0 -10
  307. package/default-pages/retro_game_starter/page.json +0 -12
  308. package/default-pages/sidebar_page/page.html +0 -51
  309. package/default-pages/sidebar_page/page.json +0 -10
  310. package/default-pages/two-panel_page/page.html +0 -68
  311. package/default-pages/two-panel_page/page.json +0 -10
@@ -1,3 +1,5 @@
1
+ import path from 'path';
2
+ import crypto from 'crypto';
1
3
  import { Application } from 'express';
2
4
  import { SynthOSConfig } from '../init';
3
5
  import { loadSettings, saveSettings } from '../settings';
@@ -6,10 +8,87 @@ import {
6
8
  ConnectorSummary,
7
9
  ConnectorDetail,
8
10
  ConnectorCallRequest,
11
+ ConnectorDefinition,
9
12
  ConnectorOAuthConfig
10
13
  } from '../connectors';
14
+ import { createMediaCache } from './mediaCache';
15
+
16
+ function base64url(buf: Buffer): string {
17
+ return buf.toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
18
+ }
19
+
20
+ function generateCodeVerifier(): string {
21
+ return base64url(crypto.randomBytes(32));
22
+ }
23
+
24
+ function generateCodeChallenge(verifier: string): string {
25
+ return base64url(crypto.createHash('sha256').update(verifier).digest());
26
+ }
27
+
28
+ interface TokenResponse {
29
+ access_token: string;
30
+ refresh_token?: string;
31
+ token_type?: string;
32
+ expires_in?: number;
33
+ scope?: string;
34
+ }
35
+
36
+ /**
37
+ * RFC 6749 §4.1.3 / §6: token endpoint requires application/x-www-form-urlencoded POST.
38
+ * Public clients (no client_secret) authenticate via PKCE alone; confidential clients send both.
39
+ */
40
+ async function exchangeOAuthToken(
41
+ tokenUrl: string,
42
+ params: Record<string, string | undefined>
43
+ ): Promise<{ ok: boolean; data?: TokenResponse; errorText?: string }> {
44
+ const body = new URLSearchParams();
45
+ for (const [k, v] of Object.entries(params)) {
46
+ if (typeof v === 'string' && v.length > 0) body.set(k, v);
47
+ }
48
+ const res = await fetch(tokenUrl, {
49
+ method: 'POST',
50
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Accept': 'application/json' },
51
+ body: body.toString()
52
+ });
53
+ if (!res.ok) {
54
+ return { ok: false, errorText: await res.text() };
55
+ }
56
+ return { ok: true, data: await res.json() as TokenResponse };
57
+ }
58
+
59
+ /**
60
+ * Refresh an expired access token. Buffer rotates refresh tokens — always save the latest.
61
+ * Returns the updated config slice (caller persists). Returns null on failure.
62
+ */
63
+ async function refreshAccessToken(
64
+ def: ConnectorDefinition,
65
+ cfg: ConnectorOAuthConfig
66
+ ): Promise<Partial<ConnectorOAuthConfig> | null> {
67
+ if (!def.tokenUrl || !cfg.refreshToken || !cfg.clientId) return null;
68
+ const result = await exchangeOAuthToken(def.tokenUrl, {
69
+ grant_type: 'refresh_token',
70
+ refresh_token: cfg.refreshToken,
71
+ client_id: cfg.clientId,
72
+ client_secret: cfg.clientSecret
73
+ });
74
+ if (!result.ok || !result.data) {
75
+ console.error(`Token refresh failed for "${def.id}":`, result.errorText);
76
+ return null;
77
+ }
78
+ const data = result.data;
79
+ return {
80
+ accessToken: data.access_token,
81
+ apiKey: data.access_token,
82
+ refreshToken: data.refresh_token ?? cfg.refreshToken,
83
+ expiresAt: data.expires_in ? Date.now() + data.expires_in * 1000 : 0
84
+ };
85
+ }
11
86
 
12
87
  export function useConnectorRoutes(config: SynthOSConfig, app: Application): void {
88
+ const mediaCache = createMediaCache({
89
+ storage: config.storageProvider,
90
+ cacheRoot: path.join(config.pagesFolder, 'cache'),
91
+ });
13
92
 
14
93
  // GET /api/connectors — List connectors (minimal summaries)
15
94
  // Also handles POST /api/connectors — Proxy call (see below)
@@ -172,8 +251,14 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
172
251
 
173
252
  const settings = await loadSettings(config);
174
253
  const cfg = (settings.connectors ?? {})[id] as ConnectorOAuthConfig | undefined;
175
- if (!cfg?.clientId || !cfg?.clientSecret) {
176
- res.status(400).json({ error: 'Client ID and Client Secret must be saved before authorizing' });
254
+ // PKCE-only public clients may register without a client_secret. Require clientId always;
255
+ // require clientSecret only for non-PKCE flows.
256
+ if (!cfg?.clientId) {
257
+ res.status(400).json({ error: 'Client ID must be saved before authorizing' });
258
+ return;
259
+ }
260
+ if (!def.usePkce && !cfg.clientSecret) {
261
+ res.status(400).json({ error: 'Client Secret must be saved before authorizing' });
177
262
  return;
178
263
  }
179
264
 
@@ -183,10 +268,22 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
183
268
  const authUrl = new URL(def.authorizationUrl!);
184
269
  authUrl.searchParams.set('client_id', cfg.clientId);
185
270
  authUrl.searchParams.set('redirect_uri', redirectUri);
186
- authUrl.searchParams.set('scope', (def.scopes ?? []).join(','));
271
+ authUrl.searchParams.set('scope', (def.scopes ?? []).join(def.scopeSeparator ?? ','));
187
272
  authUrl.searchParams.set('state', state);
188
273
  authUrl.searchParams.set('response_type', 'code');
189
274
 
275
+ // PKCE: generate verifier+challenge, persist verifier for the /callback step
276
+ if (def.usePkce) {
277
+ const codeVerifier = generateCodeVerifier();
278
+ const codeChallenge = generateCodeChallenge(codeVerifier);
279
+ authUrl.searchParams.set('code_challenge', codeChallenge);
280
+ authUrl.searchParams.set('code_challenge_method', 'S256');
281
+
282
+ const existing = settings.connectors ?? {};
283
+ const updated = { ...existing, [id]: { ...cfg, codeVerifier } };
284
+ await saveSettings(config, { connectors: updated as typeof existing });
285
+ }
286
+
190
287
  res.redirect(authUrl.toString());
191
288
  } catch (err: unknown) {
192
289
  console.error(err);
@@ -222,38 +319,45 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
222
319
 
223
320
  const settings = await loadSettings(config);
224
321
  const cfg = (settings.connectors ?? {})[connectorId] as ConnectorOAuthConfig | undefined;
225
- if (!cfg?.clientId || !cfg?.clientSecret) {
322
+ if (!cfg?.clientId) {
323
+ res.status(400).json({ error: 'Client credentials not found' });
324
+ return;
325
+ }
326
+ if (!def.usePkce && !cfg.clientSecret) {
226
327
  res.status(400).json({ error: 'Client credentials not found' });
227
328
  return;
228
329
  }
229
330
 
230
331
  const redirectUri = `${req.protocol}://${req.get('host')}/api/connectors/callback`;
231
332
 
232
- // Step 1: Exchange code for short-lived token
233
- const tokenUrl = new URL(def.tokenUrl!);
234
- tokenUrl.searchParams.set('client_id', cfg.clientId);
235
- tokenUrl.searchParams.set('client_secret', cfg.clientSecret);
236
- tokenUrl.searchParams.set('redirect_uri', redirectUri);
237
- tokenUrl.searchParams.set('code', code);
238
- tokenUrl.searchParams.set('grant_type', 'authorization_code');
239
-
240
- const tokenRes = await fetch(tokenUrl.toString());
241
- if (!tokenRes.ok) {
242
- const text = await tokenRes.text();
243
- console.error('Token exchange failed:', text);
333
+ // Step 1: Exchange code for short-lived token (RFC 6749 §4.1.3 — POST + form body)
334
+ const tokenResult = await exchangeOAuthToken(def.tokenUrl!, {
335
+ grant_type: 'authorization_code',
336
+ code,
337
+ redirect_uri: redirectUri,
338
+ client_id: cfg.clientId,
339
+ // Public PKCE clients omit client_secret; confidential clients send both.
340
+ client_secret: cfg.clientSecret,
341
+ // PKCE: include verifier when one was stored at /authorize
342
+ code_verifier: def.usePkce ? cfg.codeVerifier : undefined
343
+ });
344
+ if (!tokenResult.ok || !tokenResult.data) {
345
+ console.error('Token exchange failed:', tokenResult.errorText);
244
346
  res.redirect(`/settings?tab=connectors&error=${encodeURIComponent('Token exchange failed')}`);
245
347
  return;
246
348
  }
247
- const tokenData = await tokenRes.json() as { access_token: string; token_type?: string; expires_in?: number };
349
+ const tokenData = tokenResult.data;
248
350
  let accessToken = tokenData.access_token;
351
+ let refreshToken = tokenData.refresh_token;
249
352
  let expiresAt = tokenData.expires_in ? Date.now() + tokenData.expires_in * 1000 : 0;
250
353
 
251
- // Step 2: For Instagram — exchange for long-lived token
354
+ // Step 2: For Instagram — exchange for long-lived token (Facebook is non-PKCE,
355
+ // so clientSecret is guaranteed by the earlier non-PKCE branch validation)
252
356
  if (connectorId === 'instagram') {
253
357
  const llUrl = new URL('https://graph.facebook.com/v21.0/oauth/access_token');
254
358
  llUrl.searchParams.set('grant_type', 'fb_exchange_token');
255
359
  llUrl.searchParams.set('client_id', cfg.clientId);
256
- llUrl.searchParams.set('client_secret', cfg.clientSecret);
360
+ llUrl.searchParams.set('client_secret', cfg.clientSecret!);
257
361
  llUrl.searchParams.set('fb_exchange_token', accessToken);
258
362
 
259
363
  const llRes = await fetch(llUrl.toString());
@@ -292,22 +396,22 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
292
396
  }
293
397
  }
294
398
 
295
- // Step 4: Save tokens to settings
399
+ // Step 4: Save tokens to settings. Drop codeVerifier — it's single-use per RFC 7636.
296
400
  const existing = settings.connectors ?? {};
297
401
  const prev = (existing[connectorId] as ConnectorOAuthConfig) ?? { apiKey: '', enabled: false };
298
- const updated = {
299
- ...existing,
300
- [connectorId]: {
301
- ...prev,
302
- apiKey: prev.apiKey || accessToken,
303
- accessToken,
304
- expiresAt,
305
- userId,
306
- accountName,
307
- enabled: true
308
- }
402
+ const next: ConnectorOAuthConfig = {
403
+ ...prev,
404
+ apiKey: prev.apiKey || accessToken,
405
+ accessToken,
406
+ expiresAt,
407
+ userId,
408
+ accountName,
409
+ enabled: true
309
410
  };
310
- await saveSettings(config, { connectors: updated });
411
+ if (refreshToken) next.refreshToken = refreshToken;
412
+ delete next.codeVerifier;
413
+ const updatedConnectors = { ...existing, [connectorId]: next };
414
+ await saveSettings(config, { connectors: updatedConnectors as typeof existing });
311
415
 
312
416
  res.redirect(`/settings?tab=connectors&connected=${encodeURIComponent(connectorId)}`);
313
417
  } catch (err: unknown) {
@@ -341,8 +445,22 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
341
445
  res.status(400).json({ error: `Connector "${request.connector}" is not configured or not enabled` });
342
446
  return;
343
447
  }
344
- // Check token expiry
345
- if (oauthCfg.expiresAt && oauthCfg.expiresAt < Date.now()) {
448
+ // Auto-refresh if expired (or expiring within 60s) and a refresh_token is available.
449
+ // Skip for connectors without a refresh_token (e.g. Instagram long-lived) fall through to the 401.
450
+ const expiringSoon = oauthCfg.expiresAt && oauthCfg.expiresAt < Date.now() + 60_000;
451
+ if (expiringSoon && oauthCfg.refreshToken) {
452
+ const refreshed = await refreshAccessToken(def, oauthCfg);
453
+ if (refreshed) {
454
+ const existing = settings.connectors ?? {};
455
+ const merged = { ...oauthCfg, ...refreshed };
456
+ const updated = { ...existing, [request.connector]: merged };
457
+ await saveSettings(config, { connectors: updated as typeof existing });
458
+ Object.assign(oauthCfg, refreshed);
459
+ } else {
460
+ res.status(401).json({ error: `Access token for "${request.connector}" expired and refresh failed. Please re-authorize in Settings > Connectors.` });
461
+ return;
462
+ }
463
+ } else if (expiringSoon) {
346
464
  res.status(401).json({ error: `Access token for "${request.connector}" has expired. Please re-authorize in Settings > Connectors.` });
347
465
  return;
348
466
  }
@@ -353,6 +471,29 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
353
471
  }
354
472
  }
355
473
 
474
+ // Detect TTS calls eligible for audio caching
475
+ const isTts = request.connector === 'elevenlabs'
476
+ && request.method.toUpperCase() === 'POST'
477
+ && request.path.startsWith('/v1/text-to-speech/');
478
+ const cacheEnabled = settings.cache?.enabled !== false;
479
+ const noCache = req.headers['x-no-cache'] === 'true' || req.query.nocache === '1';
480
+
481
+ // Check audio cache for TTS requests
482
+ if (isTts && cacheEnabled && !noCache) {
483
+ const voiceId = request.path.split('/v1/text-to-speech/')[1]?.split('?')[0] ?? '';
484
+ const outputFormat = (request.query?.output_format) ?? 'mp3_44100_128';
485
+ const text = typeof request.body === 'object' && request.body !== null
486
+ ? (request.body as Record<string, unknown>).text ?? ''
487
+ : '';
488
+ const cacheKey = `audio:v1:${request.connector}:${voiceId}:${outputFormat}:${text}`;
489
+ const cached = await mediaCache.get('audio', cacheKey);
490
+ if (cached.hit) {
491
+ res.set('Content-Type', cached.contentType);
492
+ res.send(cached.buffer);
493
+ return;
494
+ }
495
+ }
496
+
356
497
  // Build URL — join baseUrl path with request path to avoid
357
498
  // absolute paths (e.g. "/me/accounts") replacing the base path.
358
499
  // Split path from inline query string first — assigning a '?' to
@@ -428,6 +569,20 @@ export function useConnectorRoutes(config: SynthOSConfig, app: Application): voi
428
569
  // Forward content-disposition if present (e.g. file downloads)
429
570
  const cd = upstream.headers.get('content-disposition');
430
571
  if (cd) res.set('Content-Disposition', cd);
572
+
573
+ // Cache TTS audio responses
574
+ if (isTts && cacheEnabled) {
575
+ const voiceId = request.path.split('/v1/text-to-speech/')[1]?.split('?')[0] ?? '';
576
+ const outputFormat = (request.query?.output_format) ?? 'mp3_44100_128';
577
+ const text = typeof request.body === 'object' && request.body !== null
578
+ ? (request.body as Record<string, unknown>).text ?? ''
579
+ : '';
580
+ const cacheKey = `audio:v1:${request.connector}:${voiceId}:${outputFormat}:${text}`;
581
+ await mediaCache.put('audio', cacheKey, buffer, ct || 'audio/mpeg', {
582
+ connector: request.connector, voiceId, outputFormat, text,
583
+ });
584
+ }
585
+
431
586
  res.send(buffer);
432
587
  }
433
588
  } catch (err: unknown) {
@@ -1,116 +1,198 @@
1
- import { Application, Response } from 'express';
2
- import { SynthOSConfig } from "../init";
3
- import path from "path";
4
- import { v4 } from "uuid";
5
- import { clearCachedScripts } from '../scripts';
6
-
7
- export function useDataRoutes(config: SynthOSConfig, app: Application): void {
8
- app.get('/api/data/:page/:table', (req, res) => handleList(config, req.params.page, req.params.table, req.query, res));
9
- app.get('/api/data/:page/:table/:id', (req, res) => handleGet(config, req.params.page, req.params.table, req.params.id, res));
10
- app.post('/api/data/:page/:table', (req, res) => handleUpsert(config, req.params.page, req.params.table, req.body, res));
11
- app.delete('/api/data/:page/:table/:id', (req, res) => handleDelete(config, req.params.page, req.params.table, req.params.id, res));
12
- }
13
-
14
- // ---------------------------------------------------------------------------
15
- // Route handlers
16
- // ---------------------------------------------------------------------------
17
-
18
- async function handleList(config: SynthOSConfig, page: string, table: string, query: Record<string, any>, res: Response): Promise<void> {
19
- const sp = config.storageProvider;
20
- const folder = tableFolder(config, page, table);
21
- if (!(await sp.checkIfExists(folder))) {
22
- res.status(404).json({ error: 'table_not_found', page, table });
23
- return;
24
- }
25
-
26
- const ids = (await sp.listFiles(folder)).filter(f => f.endsWith('.json')).map(f => f.replace('.json', ''));
27
-
28
- const rows: Record<string, any>[] = [];
29
- for (const id of ids) {
30
- const file = recordFile(folder, id);
31
- try {
32
- const row = JSON.parse(await sp.loadFile(file));
33
- row.id = id;
34
- rows.push(row);
35
- } catch (err: unknown) {
36
- console.error(err);
37
- }
38
- }
39
-
40
- // Paginate when limit is provided
41
- const limitParam = typeof query.limit === 'string' ? parseInt(query.limit, 10) : NaN;
42
- if (!isNaN(limitParam) && limitParam > 0) {
43
- const offset = Math.max(0, typeof query.offset === 'string' ? parseInt(query.offset, 10) || 0 : 0);
44
- const items = rows.slice(offset, offset + limitParam);
45
- res.json({ items, total: rows.length, offset, limit: limitParam, hasMore: offset + limitParam < rows.length });
46
- } else {
47
- res.json(rows);
48
- }
49
- }
50
-
51
- async function handleGet(config: SynthOSConfig, page: string, table: string, id: string, res: Response): Promise<void> {
52
- const sp = config.storageProvider;
53
- const folder = tableFolder(config, page, table);
54
- if (!(await sp.checkIfExists(folder))) {
55
- res.status(404).json({ error: 'table_not_found', page, table });
56
- return;
57
- }
58
-
59
- const file = recordFile(folder, id);
60
- try {
61
- const row = JSON.parse(await sp.loadFile(file));
62
- row.id = id;
63
- res.json(row);
64
- } catch (err: unknown) {
65
- res.json({});
66
- }
67
- }
68
-
69
- async function handleUpsert(config: SynthOSConfig, page: string, table: string, body: any, res: Response): Promise<void> {
70
- const sp = config.storageProvider;
71
- const id = body.id ?? v4();
72
- const folder = tableFolder(config, page, table);
73
- const file = recordFile(folder, id);
74
- try {
75
- const row = { ...body, id };
76
- await sp.ensureFolderExists(folder);
77
- await sp.saveFile(file, JSON.stringify(row, null, 4));
78
- if (table === 'scripts') {
79
- clearCachedScripts();
80
- }
81
- res.json(row);
82
- } catch (err: unknown) {
83
- console.error(err);
84
- res.status(500).send((err as Error).message);
85
- }
86
- }
87
-
88
- async function handleDelete(config: SynthOSConfig, page: string, table: string, id: string, res: Response): Promise<void> {
89
- const sp = config.storageProvider;
90
- const folder = tableFolder(config, page, table);
91
- const file = recordFile(folder, id);
92
- try {
93
- if (await sp.checkIfExists(file)) {
94
- await sp.deleteFile(file);
95
- if (table === 'scripts') {
96
- clearCachedScripts();
97
- }
98
- }
99
- res.json({ success: true });
100
- } catch (err: unknown) {
101
- console.error(err);
102
- res.status(500).send((err as Error).message);
103
- }
104
- }
105
-
106
- // ---------------------------------------------------------------------------
107
- // Helpers
108
- // ---------------------------------------------------------------------------
109
-
110
- function tableFolder(config: SynthOSConfig, page: string, table: string): string {
111
- return path.join(config.pagesFolder, 'pages', page, table);
112
- }
113
-
114
- function recordFile(folder: string, id: string): string {
115
- return path.join(folder, `${id}.json`);
116
- }
1
+ import { Application, Response } from 'express';
2
+ import { SynthOSConfig } from "../init";
3
+ import path from "path";
4
+ import { v4 } from "uuid";
5
+ import { clearCachedScripts } from '../scripts';
6
+ import {
7
+ deleteSchema,
8
+ isValidSchemaPayload,
9
+ listTables,
10
+ loadSchema,
11
+ mergeSchema,
12
+ newSchemaWrapper,
13
+ saveSchema,
14
+ updateSchemaWrapper,
15
+ MergeMode,
16
+ } from "./sharedTableSchema";
17
+
18
+ export function useDataRoutes(config: SynthOSConfig, app: Application): void {
19
+ // Schema sidecar + table-list endpoints registered before the generic
20
+ // table/record routes so the literal `_schema` / `_tables` segments
21
+ // aren't captured by `:table` / `:table/:id`.
22
+ app.get('/api/data/:page/_tables', (req, res) => handleListTables(config, req.params.page, res));
23
+ app.get('/api/data/:page/:table/_schema', (req, res) => handleGetSchema(config, req.params.page, req.params.table, res));
24
+ app.put('/api/data/:page/:table/_schema', (req, res) => handlePutSchema(config, req.params.page, req.params.table, req.query, req.body, res));
25
+ app.delete('/api/data/:page/:table/_schema', (req, res) => handleDeleteSchema(config, req.params.page, req.params.table, res));
26
+
27
+ app.get('/api/data/:page/:table', (req, res) => handleList(config, req.params.page, req.params.table, req.query, res));
28
+ app.get('/api/data/:page/:table/:id', (req, res) => handleGet(config, req.params.page, req.params.table, req.params.id, res));
29
+ app.post('/api/data/:page/:table', (req, res) => handleUpsert(config, req.params.page, req.params.table, req.body, res));
30
+ app.delete('/api/data/:page/:table/:id', (req, res) => handleDelete(config, req.params.page, req.params.table, req.params.id, res));
31
+ }
32
+
33
+ // ---------------------------------------------------------------------------
34
+ // Route handlers — records
35
+ // ---------------------------------------------------------------------------
36
+
37
+ async function handleList(config: SynthOSConfig, page: string, table: string, query: Record<string, any>, res: Response): Promise<void> {
38
+ const sp = config.storageProvider;
39
+ const folder = tableFolder(config, page, table);
40
+ if (!(await sp.checkIfExists(folder))) {
41
+ res.status(404).json({ error: 'table_not_found', page, table });
42
+ return;
43
+ }
44
+
45
+ const ids = (await sp.listFiles(folder)).filter(f => f.endsWith('.json')).map(f => f.replace('.json', ''));
46
+
47
+ const rows: Record<string, any>[] = [];
48
+ for (const id of ids) {
49
+ const file = recordFile(folder, id);
50
+ try {
51
+ const row = JSON.parse(await sp.loadFile(file));
52
+ row.id = id;
53
+ rows.push(row);
54
+ } catch (err: unknown) {
55
+ console.error(err);
56
+ }
57
+ }
58
+
59
+ // Paginate when limit is provided
60
+ const limitParam = typeof query.limit === 'string' ? parseInt(query.limit, 10) : NaN;
61
+ if (!isNaN(limitParam) && limitParam > 0) {
62
+ const offset = Math.max(0, typeof query.offset === 'string' ? parseInt(query.offset, 10) || 0 : 0);
63
+ const items = rows.slice(offset, offset + limitParam);
64
+ res.json({ items, total: rows.length, offset, limit: limitParam, hasMore: offset + limitParam < rows.length });
65
+ } else {
66
+ res.json(rows);
67
+ }
68
+ }
69
+
70
+ async function handleGet(config: SynthOSConfig, page: string, table: string, id: string, res: Response): Promise<void> {
71
+ const sp = config.storageProvider;
72
+ const folder = tableFolder(config, page, table);
73
+ if (!(await sp.checkIfExists(folder))) {
74
+ res.status(404).json({ error: 'table_not_found', page, table });
75
+ return;
76
+ }
77
+
78
+ const file = recordFile(folder, id);
79
+ try {
80
+ const row = JSON.parse(await sp.loadFile(file));
81
+ row.id = id;
82
+ res.json(row);
83
+ } catch (err: unknown) {
84
+ res.json({});
85
+ }
86
+ }
87
+
88
+ async function handleUpsert(config: SynthOSConfig, page: string, table: string, body: any, res: Response): Promise<void> {
89
+ const sp = config.storageProvider;
90
+ const id = body.id ?? v4();
91
+ const folder = tableFolder(config, page, table);
92
+ const file = recordFile(folder, id);
93
+ try {
94
+ const row = { ...body, id };
95
+ await sp.ensureFolderExists(folder);
96
+ await sp.saveFile(file, JSON.stringify(row, null, 4));
97
+ if (table === 'scripts') {
98
+ clearCachedScripts();
99
+ }
100
+ res.json(row);
101
+ } catch (err: unknown) {
102
+ console.error(err);
103
+ res.status(500).send((err as Error).message);
104
+ }
105
+ }
106
+
107
+ async function handleDelete(config: SynthOSConfig, page: string, table: string, id: string, res: Response): Promise<void> {
108
+ const sp = config.storageProvider;
109
+ const folder = tableFolder(config, page, table);
110
+ const file = recordFile(folder, id);
111
+ try {
112
+ if (await sp.checkIfExists(file)) {
113
+ await sp.deleteFile(file);
114
+ if (table === 'scripts') {
115
+ clearCachedScripts();
116
+ }
117
+ }
118
+ res.json({ success: true });
119
+ } catch (err: unknown) {
120
+ console.error(err);
121
+ res.status(500).send((err as Error).message);
122
+ }
123
+ }
124
+
125
+ // ---------------------------------------------------------------------------
126
+ // Route handlers — schema sidecar
127
+ // ---------------------------------------------------------------------------
128
+
129
+ async function handleGetSchema(config: SynthOSConfig, page: string, table: string, res: Response): Promise<void> {
130
+ const wrapper = await loadSchema(config, pageNamespace(config, page), table);
131
+ if (!wrapper) {
132
+ res.status(404).json({ error: 'schema_not_found', page, table });
133
+ return;
134
+ }
135
+ res.json(wrapper);
136
+ }
137
+
138
+ async function handlePutSchema(
139
+ config: SynthOSConfig,
140
+ page: string,
141
+ table: string,
142
+ query: Record<string, any>,
143
+ body: any,
144
+ res: Response,
145
+ ): Promise<void> {
146
+ const incoming = body && typeof body === 'object' && body.schema ? body.schema : body;
147
+ if (!isValidSchemaPayload(incoming)) {
148
+ res.status(400).json({ error: 'invalid_schema', message: 'Body must be a JSON Schema object (or { schema: ... } wrapper).' });
149
+ return;
150
+ }
151
+ const merge: MergeMode = query.merge === 'replace' ? 'replace' : 'additive';
152
+ const definedBy = typeof body?.definedBy === 'string' ? body.definedBy : undefined;
153
+ const namespace = pageNamespace(config, page);
154
+ const existing = await loadSchema(config, namespace, table);
155
+ const { merged, conflicts } = mergeSchema(existing?.schema, incoming, merge);
156
+ if (conflicts.length > 0) {
157
+ res.status(409).json({ error: 'schema_conflict', conflicts });
158
+ return;
159
+ }
160
+ const now = new Date().toISOString();
161
+ const wrapper = existing
162
+ ? updateSchemaWrapper(existing, merged, now, definedBy)
163
+ : newSchemaWrapper(merged, now, definedBy);
164
+ await saveSchema(config, namespace, table, wrapper);
165
+ res.json(wrapper);
166
+ }
167
+
168
+ async function handleDeleteSchema(config: SynthOSConfig, page: string, table: string, res: Response): Promise<void> {
169
+ await deleteSchema(config, pageNamespace(config, page), table);
170
+ res.status(204).end();
171
+ }
172
+
173
+ /**
174
+ * Subfolders under `<pagesFolder>/pages/<page>/` that are NOT data tables.
175
+ * `files` is the synthos.files namespace; it must not appear in the table list.
176
+ */
177
+ const RESERVED_PAGE_SUBDIRS: ReadonlySet<string> = new Set(['files']);
178
+
179
+ async function handleListTables(config: SynthOSConfig, page: string, res: Response): Promise<void> {
180
+ const tables = await listTables(config, pageNamespace(config, page), RESERVED_PAGE_SUBDIRS);
181
+ res.json({ tables });
182
+ }
183
+
184
+ // ---------------------------------------------------------------------------
185
+ // Helpers
186
+ // ---------------------------------------------------------------------------
187
+
188
+ function pageNamespace(config: SynthOSConfig, page: string): string {
189
+ return path.join(config.pagesFolder, 'pages', page);
190
+ }
191
+
192
+ function tableFolder(config: SynthOSConfig, page: string, table: string): string {
193
+ return path.join(pageNamespace(config, page), table);
194
+ }
195
+
196
+ function recordFile(folder: string, id: string): string {
197
+ return path.join(folder, `${id}.json`);
198
+ }