pi-sap-aicore 0.3.2 → 0.3.3

package/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.3] - 2026-07-01
+
+### Added
+
+- Added `npm run validate:foundation`, a live SAP AI Core validation matrix for
+  direct foundation executables. It verifies text generation, real tool execution
+  side effects, and image input across Azure OpenAI, AWS Bedrock, and GCP Vertex
+  AI foundation routes.
+
+### Changed
+
+- Raised SAP AI SDK dependency floors to `^2.12.0` so fresh installs prefer the
+  patched transitive `axios` / `form-data` tree that resolves the npm audit
+  advisory for `form-data <4.0.6`.
+
 ## [0.3.2] - 2026-07-01
 
 ### Fixed
@@ -146,7 +161,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `reasoning_effort` for OpenAI).
 - MIT license and npm packaging.
 
-[Unreleased]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.3.2...HEAD
+[Unreleased]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.3.3...HEAD
+[0.3.3]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.3.2...v0.3.3
 [0.3.2]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.3.1...v0.3.2
 [0.3.1]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/ttiimmaahh/pi-sap-aicore/compare/v0.2.2...v0.3.0

package/README.md

@@ -160,6 +160,26 @@ there. An id with no matching deployment 404s at call time. Run
 `/sap-models discover` in pi (or `node scripts/list-sap-models.mjs` from this
 repo) to see what your tenant actually deploys.
 
+### Live foundation validation
+
+To validate normal coding-agent behavior across the three direct foundation
+executables, run the live smoke-test matrix from this repo:
+
+```bash
+npm run validate:foundation
+```
+
+The script uses `pi --no-extensions -e ./index.ts` so it tests the local checkout,
+not a globally installed package. It makes real SAP model calls and validates:
+
+- text generation
+- real tool execution via side-effect files
+- image input smoke tests
+
+Default models are `gpt-5.5`, `anthropic--claude-4.8-opus`, and
+`gemini-3.5-flash`. Override them with `GPT_MODEL`, `BEDROCK_MODEL`, and
+`VERTEX_MODEL`; set `SKIP_IMAGE=1` to skip vision tests.
+
 ## Models
 
 The model list is composed of three sources, merged at startup:
@@ -450,9 +470,10 @@ npmjs.com:
 │   └── publish.yml           # tag-driven npm publish via OIDC trusted publishing
 ├── index.ts                  # ExtensionAPI factory + registerProvider calls (both providers)
 ├── scripts/
-│   ├── update-models.mjs     # maintainer script: fetches models.dev, writes models-snapshot.json
-│   ├── list-sap-models.mjs   # lists models your tenant actually deploys (diff vs snapshot)
-│   └── diagnose-streaming.mjs # probes orchestration streaming support per model
+│   ├── update-models.mjs                    # maintainer script: fetches models.dev, writes models-snapshot.json
+│   ├── list-sap-models.mjs                  # lists models your tenant actually deploys (diff vs snapshot)
+│   ├── diagnose-streaming.mjs               # probes orchestration streaming support per model
+│   └── validate-foundation-executables.mjs  # live text/tool/image smoke tests for direct foundation executables
 └── src/
     ├── auth.ts                  # service-key validation + pi oauth registration
     ├── model-catalog.ts         # loads snapshot/cache/overlay and adapts models.dev metadata

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-sap-aicore",
-  "version": "0.3.2",
+  "version": "0.3.3",
   "description": "SAP AI Core (orchestration + foundation) provider for the pi coding agent",
   "license": "MIT",
   "author": "Tim Pearson (https://github.com/ttiimmaahh)",
@@ -31,12 +31,13 @@
   },
   "scripts": {
     "update-models": "node scripts/update-models.mjs",
+    "validate:foundation": "node scripts/validate-foundation-executables.mjs",
     "prepublishOnly": "tsc --noEmit"
   },
   "dependencies": {
-    "@sap-ai-sdk/ai-api": "^2.10.0",
-    "@sap-ai-sdk/foundation-models": "^2.10.0",
-    "@sap-ai-sdk/orchestration": "^2.10.0"
+    "@sap-ai-sdk/ai-api": "^2.12.0",
+    "@sap-ai-sdk/foundation-models": "^2.12.0",
+    "@sap-ai-sdk/orchestration": "^2.12.0"
   },
   "peerDependencies": {
     "@earendil-works/pi-ai": "*",

package/scripts/validate-foundation-executables.mjs

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+// Live validation for SAP AI Core direct foundation executables.
+//
+// This script makes real, billed model calls through pi using the local extension.
+// It validates the three high-value foundation executable families:
+//   - azure-openai  → GPT/OpenAI models
+//   - aws-bedrock   → Anthropic/Claude models
+//   - gcp-vertexai  → Gemini models
+//
+// Usage from repo root:
+//   node scripts/validate-foundation-executables.mjs
+//
+// Optional model overrides:
+//   GPT_MODEL=gpt-5.5 \
+//   BEDROCK_MODEL=anthropic--claude-4.8-opus \
+//   VERTEX_MODEL=gemini-3.5-flash \
+//   node scripts/validate-foundation-executables.mjs
+//
+// Optional: SKIP_IMAGE=1 to skip vision tests.
+
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const ROOT = new URL("..", import.meta.url).pathname;
+const PI = process.env.PI_BIN ?? "pi";
+const SKIP_IMAGE = process.env.SKIP_IMAGE === "1";
+
+const SCENARIOS = [
+	{
+		name: "azure-openai/gpt",
+		model: process.env.GPT_MODEL ?? "gpt-5.5",
+		marker: "GPT_TOOL_OK",
+	},
+	{
+		name: "aws-bedrock/anthropic",
+		model: process.env.BEDROCK_MODEL ?? "anthropic--claude-4.8-opus",
+		marker: "BEDROCK_TOOL_OK",
+	},
+	{
+		name: "gcp-vertexai/gemini",
+		model: process.env.VERTEX_MODEL ?? "gemini-3.5-flash",
+		marker: "VERTEX_TOOL_OK",
+	},
+];
+
+const tinyPng = Buffer.from(
+	"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=",
+	"base64",
+);
+
+function runPi(args, options = {}) {
+	const result = spawnSync(PI, args, {
+		cwd: ROOT,
+		encoding: "utf8",
+		maxBuffer: 10 * 1024 * 1024,
+		...options,
+	});
+	return {
+		status: result.status ?? 1,
+		stdout: result.stdout ?? "",
+		stderr: result.stderr ?? "",
+		combined: `${result.stdout ?? ""}${result.stderr ?? ""}`,
+	};
+}
+
+function baseArgs(model) {
+	return [
+		"--no-extensions",
+		"-e",
+		"./index.ts",
+		"--model",
+		`sap-aicore-foundation/${model}`,
+		"--no-context-files",
+		"--no-skills",
+		"-p",
+	];
+}
+
+function assert(condition, message, detail) {
+	if (condition) return;
+	console.error(`\n❌ ${message}`);
+	if (detail) console.error(detail);
+	process.exitCode = 1;
+	throw new Error(message);
+}
+
+function truncate(s, max = 2000) {
+	return s.length > max ? `${s.slice(0, max)}…[+${s.length - max} chars]` : s;
+}
+
+function validateText(scenario) {
+	const prompt = "Reply with exactly the single token: TEXT_OK";
+	const result = runPi([...baseArgs(scenario.model), prompt]);
+	assert(
+		result.status === 0,
+		`${scenario.name} text scenario exited ${result.status}`,
+		truncate(result.combined),
+	);
+	assert(
+		/TEXT_OK/.test(result.combined),
+		`${scenario.name} text scenario did not contain TEXT_OK`,
+		truncate(result.combined),
+	);
+	console.log(`  ✓ text generation`);
+}
+
+function validateToolUse(scenario, workDir) {
+	const file = join(workDir, `${scenario.name.replaceAll("/", "-")}.txt`);
+	const prompt =
+		`Use the bash tool to run exactly: printf ${scenario.marker} > ${file}. ` +
+		"Then say done.";
+	const result = runPi([...baseArgs(scenario.model), prompt]);
+	assert(
+		result.status === 0,
+		`${scenario.name} tool scenario exited ${result.status}`,
+		truncate(result.combined),
+	);
+	let actual = "";
+	try {
+		actual = readFileSync(file, "utf8");
+	} catch {
+		// handled below
+	}
+	assert(
+		actual === scenario.marker,
+		`${scenario.name} tool scenario did not create expected side-effect`,
+		`expected file ${file} to contain ${scenario.marker}\nmodel output:\n${truncate(result.combined)}`,
+	);
+	console.log(`  ✓ tool execution side effect`);
+}
+
+function validateImage(scenario, workDir) {
+	const imagePath = join(workDir, "tiny.png");
+	writeFileSync(imagePath, tinyPng);
+	const prompt =
+		"Look at @" +
+		imagePath +
+		" and reply with exactly IMAGE_OK if you can inspect the attached image.";
+	const result = runPi([...baseArgs(scenario.model), prompt]);
+	assert(
+		result.status === 0,
+		`${scenario.name} image scenario exited ${result.status}`,
+		truncate(result.combined),
+	);
+	assert(
+		/IMAGE_OK/.test(result.combined),
+		`${scenario.name} image scenario did not contain IMAGE_OK`,
+		truncate(result.combined),
+	);
+	console.log(`  ✓ image input smoke test`);
+}
+
+const workDir = mkdtempSync(join(tmpdir(), "pi-sap-aicore-foundation-"));
+console.log(`Live SAP AI Core foundation validation`);
+console.log(`repo: ${ROOT}`);
+console.log(`workDir: ${workDir}`);
+console.log(`skip image: ${SKIP_IMAGE ? "yes" : "no"}\n`);
+
+try {
+	for (const scenario of SCENARIOS) {
+		console.log(`${scenario.name}  model=${scenario.model}`);
+		validateText(scenario);
+		validateToolUse(scenario, workDir);
+		if (!SKIP_IMAGE) validateImage(scenario, workDir);
+		console.log("");
+	}
+	console.log("✅ all foundation executable scenarios passed");
+} catch {
+	// assert() already printed details and set exitCode.
+} finally {
+	if (process.env.KEEP_VALIDATION_ARTIFACTS !== "1") {
+		rmSync(workDir, { recursive: true, force: true });
+	} else {
+		console.log(`kept artifacts: ${workDir}`);
+	}
+}