pi-apexlang 0.2.2 → 0.4.0

package/extensions/lib/apexlang-cli.mjs

@@ -4,6 +4,7 @@ import { cp, lstat, mkdir, readFile, readdir, realpath, rm, writeFile } from "no
 import { dirname, extname, isAbsolute, relative, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { stripVTControlCharacters } from "node:util";
+import { normalizeApxLineEndings } from "../../skills/apexlang/runtime/lib/common.mjs";
 
 const moduleDirectory = dirname(fileURLToPath(import.meta.url));
 
@@ -21,6 +22,8 @@ export const APEXLANG_ACTIONS = Object.freeze([
 
 const apexctlPath = resolve(APEXLANG_SKILL_ROOT, "tools/apexctl.mjs");
 const queryValidPropsPath = resolve(APEXLANG_SKILL_ROOT, "tools/query-valid-props.mjs");
+const localValidatePath = resolve(moduleDirectory, "apexlang-local-validate.mjs");
+const runtimeValidatePath = resolve(moduleDirectory, "apexlang-runtime-validate.mjs");
 const runtimeRoundtripPath = resolve(moduleDirectory, "apexlang-runtime-roundtrip.mjs");
 const sqlclPtyProxyPath = resolve(moduleDirectory, "apexlang-sqlcl-pty.py");
 const CONTROL_CHARACTER_PATTERN = /[\u0000-\u001f\u007f]/;
@@ -214,9 +217,9 @@ export function buildApexlangCommand(input) {
   }
 
   if (action === "local_validate") {
-    const args = ["apexlang", "validate", "--app-path", requirePath(input, "app_path", action)];
+    const args = ["--app-path", requirePath(input, "app_path", action)];
     addFlag(args, "--fix-vocab", input.fix_vocab);
-    return apexctl(args);
+    return { scriptPath: localValidatePath, args, prelude: [] };
   }
 
   if (action === "compiler_truth_audit") {
@@ -309,7 +312,11 @@ export function buildApexlangCommand(input) {
   addOption(args, "--apex-root", optionalPath(input, "apex_root"));
   addOption(args, "--compiler-oracle-home", optionalPath(input, "compiler_oracle_home"));
   addFlag(args, "--supporting-objects", input.supporting_objects);
-  return apexctl(args, [{ scriptPath: apexctlPath, args: probeArgs }]);
+  return {
+    scriptPath: runtimeValidatePath,
+    args,
+    prelude: [{ scriptPath: apexctlPath, args: probeArgs }]
+  };
 }
 
 export function buildApexlangImportCommand(input, importOptions = {}) {
@@ -621,7 +628,7 @@ async function clearStaleRuntimeEvidence(command, outputRoot) {
   }
 }
 
-export async function prepareRuntimeApp(input, cwd, outputRoot, prepareOptions = {}) {
+export async function prepareRuntimeApp(input, cwd, outputRoot) {
   if (!runtimeActionUsesDeployment(input)) {
     return { appPath: input.app_path ? resolve(cwd, input.app_path) : undefined, staged: false };
   }
@@ -663,10 +670,6 @@ export async function prepareRuntimeApp(input, cwd, outputRoot, prepareOptions =
     );
   }
 
-  if (topLevelWorkspace && prepareOptions.forceStage !== true) {
-    return { appPath, staged: false, workspaceSource: "workspace.name" };
-  }
-
   const stageKey = createHash("sha256")
     .update(`${appPath}\0${requestedWorkspace.toUpperCase()}`)
     .digest("hex")
@@ -701,6 +704,10 @@ export async function prepareRuntimeApp(input, cwd, outputRoot, prepareOptions =
     `${JSON.stringify(deployment, null, 2)}\n`,
     "utf8"
   );
+  // Oracle normalizes source line endings before live validation/import. Apply
+  // that same normalization only to the staged copy, before binding its digest,
+  // so validation and import approve identical bytes without changing the project.
+  await normalizeApxLineEndings(stagedAppPath);
   return {
     appPath: stagedAppPath,
     staged: true,
@@ -2118,9 +2125,7 @@ async function runCommand(input, command, options, runOptions = {}) {
   if (input.action === "workspace_probe" || command.prelude.length > 0) {
     await assertWorkspaceProbeCannotFollowSymlinks(options.cwd);
   }
-  const runtimeApp = await prepareRuntimeApp(input, options.cwd, options.outputRoot, {
-    forceStage: runOptions.forceRuntimeStage === true
-  });
+  const runtimeApp = await prepareRuntimeApp(input, options.cwd, options.outputRoot);
   const preRunAppDigest = runtimeApp.appPath && runtimeActionUsesDeployment(input)
     ? await computeApexlangAppDigest(runtimeApp.appPath)
     : undefined;
@@ -2189,7 +2194,7 @@ export async function runApexlangImport(input, options, importOptions = {}) {
     normalizedInput,
     buildApexlangImportCommand(normalizedInput, importOptions),
     options,
-    { forceRuntimeStage: true, expectedAppDigest }
+    { expectedAppDigest }
   );
   if (result.ok || importOptions.targetResolutionMode === "create-new") return result;
   return runWarningCompatibleImport(normalizedInput, result, options);
@@ -2205,6 +2210,6 @@ export function runApexlangCreateNewProof(input, options, proofOptions = {}) {
       createNewConfirmed: false
     }),
     options,
-    { forceRuntimeStage: true, expectedAppDigest }
+    { expectedAppDigest }
   );
 }

package/extensions/lib/apexlang-local-validate.mjs

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+
+import { spawnSync } from "node:child_process";
+import { mkdirSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const moduleDirectory = dirname(fileURLToPath(import.meta.url));
+const skillRoot = resolve(moduleDirectory, "../../skills/apexlang");
+const runtimeRoot = resolve(skillRoot, "runtime");
+const pythonRoot = resolve(runtimeRoot, "internal/python");
+const optimizedDslValidator = resolve(moduleDirectory, "apexlang-local-validator.py");
+
+function readOption(args, name) {
+  const index = args.indexOf(name);
+  return index >= 0 && index + 1 < args.length ? String(args[index + 1]).trim() : "";
+}
+
+function main() {
+  const args = process.argv.slice(2);
+  const appPath = readOption(args, "--app-path");
+  if (!appPath) {
+    console.error("Missing required --app-path");
+    return 1;
+  }
+
+  const outputRoot = String(process.env.APEXLANG_OUTPUT_ROOT ?? "").trim();
+  if (!outputRoot) {
+    console.error("APEXLANG_OUTPUT_ROOT is required in packaged apexlang runtime");
+    return 1;
+  }
+
+  const reportDir = resolve(outputRoot, "logs");
+  mkdirSync(reportDir, { recursive: true });
+  const fixVocab = args.includes("--fix-vocab");
+  const commands = [
+    [
+      resolve(pythonRoot, "validate_apexlang_vocab.py"),
+      [
+        "--app-path",
+        appPath,
+        "--report-path",
+        join(reportDir, "apexlang-vocab-report.json"),
+        fixVocab ? "--rewrite" : "--check-only"
+      ]
+    ],
+    [
+      optimizedDslValidator,
+      ["--report-path", join(reportDir, "apexlang-dsl-report.json"), appPath]
+    ],
+    [
+      resolve(pythonRoot, "validate_validations.py"),
+      ["--report-path", join(reportDir, "apexlang-validations-report.json"), appPath]
+    ]
+  ];
+
+  let failures = 0;
+  for (const [scriptPath, scriptArgs] of commands) {
+    const result = spawnSync("python3", [scriptPath, ...scriptArgs], {
+      cwd: process.cwd(),
+      encoding: "utf8",
+      env: {
+        ...process.env,
+        APEXLANG_EMBEDDED_TOOLS_ROOT: resolve(skillRoot, "tools"),
+        APEXLANG_PACKAGE_ROOT: skillRoot,
+        APEXLANG_RUNTIME_ROOT: runtimeRoot,
+        PYTHONDONTWRITEBYTECODE: "1"
+      }
+    });
+    if (result.stdout) process.stdout.write(result.stdout);
+    if (result.stderr) process.stderr.write(result.stderr);
+    if (typeof result.status !== "number" || result.status !== 0) failures += 1;
+  }
+
+  if (failures > 0) {
+    console.log("APEXLANG_LOCAL_CHECK_FAILED");
+    return 1;
+  }
+  console.log("APEXLANG_LOCAL_CHECK_OK");
+  return 0;
+}
+
+process.exitCode = main();

package/extensions/lib/apexlang-local-validator.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""Run Oracle's DSL validator with per-process parser acceleration.
+
+The vendored Oracle skill stays byte-for-byte unchanged. This launcher imports
+its validator, replaces only pure parsing helpers with equivalent cached
+implementations, and then calls the original command-line entry point.
+"""
+
+from __future__ import annotations
+
+from bisect import bisect_right
+from functools import lru_cache, wraps
+import importlib.util
+import os
+from pathlib import Path
+import sys
+from types import ModuleType
+from typing import Any, Callable
+
+
+class _SparseDepthIndex:
+    """Store lexical depth checkpoints only at offsets requested by validators."""
+
+    __slots__ = ("positions", "states")
+
+    def __init__(self) -> None:
+        self.positions: list[int] = [0]
+        self.states: list[tuple[int, int, bool]] = [(0, 0, False)]
+
+
+_DEPTH_INDEXES: dict[str, _SparseDepthIndex] = {}
+
+
+def _sparse_nesting_depth(text: str, idx: int) -> tuple[int, int]:
+    """Return Oracle-compatible nesting depth using the closest prior checkpoint."""
+    target_idx = min(len(text), idx) if idx >= 0 else max(0, len(text) + idx)
+    index = _DEPTH_INDEXES.setdefault(text, _SparseDepthIndex())
+    checkpoint_slot = bisect_right(index.positions, target_idx) - 1
+    start = index.positions[checkpoint_slot]
+    paren_depth, brace_depth, in_string = index.states[checkpoint_slot]
+
+    for pos in range(start, target_idx):
+        ch = text[pos]
+        if ch == '"' and (pos == 0 or text[pos - 1] != "\\"):
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if ch == "(":
+            paren_depth += 1
+        elif ch == ")":
+            paren_depth = max(0, paren_depth - 1)
+        elif ch == "{":
+            brace_depth += 1
+        elif ch == "}":
+            brace_depth = max(0, brace_depth - 1)
+
+    if start != target_idx:
+        insert_at = checkpoint_slot + 1
+        index.positions.insert(insert_at, target_idx)
+        index.states.insert(insert_at, (paren_depth, brace_depth, in_string))
+    return paren_depth, brace_depth
+
+
+def _validator_path() -> Path:
+    runtime_override = os.environ.get("APEXLANG_RUNTIME_ROOT", "").strip()
+    runtime_root = (
+        Path(runtime_override).resolve()
+        if runtime_override
+        else Path(__file__).resolve().parents[2] / "skills" / "apexlang" / "runtime"
+    )
+    return runtime_root / "internal" / "python" / "validate_apexlang.py"
+
+
+def _load_oracle_validator(path: Path) -> ModuleType:
+    sys.path.insert(0, str(path.parent))
+    spec = importlib.util.spec_from_file_location("_pi_apexlang_oracle_validator", path)
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Cannot load Oracle APEXlang validator: {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _cache_list_function(function: Callable[..., list[Any]]) -> Callable[..., list[Any]]:
+    """Cache immutable results while preserving Oracle's fresh-list return type."""
+    cached = lru_cache(maxsize=None)(lambda *args: tuple(function(*args)))
+
+    @wraps(function)
+    def wrapper(*args: Any) -> list[Any]:
+        return list(cached(*args))
+
+    return wrapper
+
+
+def _accelerate(module: ModuleType) -> None:
+    module.nesting_depth = _sparse_nesting_depth
+    for name in (
+        "find_component_blocks",
+        "find_named_brace_blocks",
+        "find_immediate_component_blocks",
+        "find_immediate_named_brace_blocks",
+    ):
+        setattr(module, name, _cache_list_function(getattr(module, name)))
+    module.block_body = lru_cache(maxsize=None)(module.block_body)
+
+
+def main(argv: list[str]) -> int:
+    validator = _load_oracle_validator(_validator_path())
+    _accelerate(validator)
+    return int(validator.main(argv))
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

package/extensions/lib/apexlang-runtime-validate.mjs

@@ -0,0 +1,76 @@
+import { dirname, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const moduleDirectory = dirname(fileURLToPath(import.meta.url));
+const packageRoot = resolve(moduleDirectory, "../../skills/apexlang");
+const runtimeRoot = resolve(packageRoot, "runtime");
+
+export async function loadPackagedRuntimeValidation() {
+  process.env.APEXLANG_PACKAGE_ROOT = packageRoot;
+  process.env.APEXLANG_RUNTIME_ROOT = runtimeRoot;
+  process.env.APEXLANG_EMBEDDED_TOOLS_ROOT = resolve(packageRoot, "tools");
+
+  const [runtime, grammar, common] = await Promise.all([
+    import(pathToFileURL(resolve(runtimeRoot, "runtime.bundle.mjs")).href),
+    import(pathToFileURL(resolve(runtimeRoot, "grammar_contract.mjs")).href),
+    import(pathToFileURL(resolve(runtimeRoot, "lib/common.mjs")).href)
+  ]);
+  // Oracle's packaged Media List gate still addresses these two helpers using
+  // source-tree paths. Redirect only their locations; keep its gate and reports.
+  const dependencies = {
+    resolveMediaListGrammarContract: (options) => runtime.resolveMediaListGrammarContract({
+      ...options,
+      buildGrammarContractFn: grammar.buildGrammarContract
+    }),
+    runTargetBuildLocalValidation: ({ appPath, componentContractPath }) => common.runCommand(
+      "python3",
+      [
+        resolve(runtimeRoot, "internal/python/validate_apexlang.py"),
+        "--component-attributes",
+        componentContractPath,
+        appPath
+      ],
+      { allowFailure: true, passthrough: false, env: { PYTHONDONTWRITEBYTECODE: "1" } }
+    )
+  };
+  return {
+    ...dependencies,
+    run: (options) => runtime.runRuntimeValidate({
+      ...options,
+      _deps: { ...dependencies, ...options._deps }
+    })
+  };
+}
+
+function readOption(args, name, fallback = "") {
+  const index = args.indexOf(name);
+  return index >= 0 && index + 1 < args.length ? args[index + 1] : fallback;
+}
+
+async function main(args) {
+  if (args[0] !== "runtime" || args[1] !== "validate") {
+    throw new Error("The pi APEXlang validation launcher accepts only runtime validate.");
+  }
+  const validation = await loadPackagedRuntimeValidation();
+  const result = await validation.run({
+    appPath: readOption(args, "--app-path"),
+    dbConnectionName: readOption(args, "--db-connection-name"),
+    executionMode: readOption(args, "--execution-mode", "auto"),
+    targetResolutionMode: readOption(args, "--target-resolution-mode", "update-existing"),
+    workspaceId: readOption(args, "--workspaceid"),
+    supportingObjects: args.includes("--supporting-objects"),
+    preflightOnly: args.includes("--preflight-only"),
+    apexRoot: readOption(args, "--apex-root"),
+    compilerOracleHome: readOption(args, "--compiler-oracle-home"),
+    artifactDir: readOption(args, "--artifact-dir"),
+    vscodeProblemsPath: readOption(args, "--vscode-problems-path"),
+    reportPath: readOption(args, "--report-path"),
+    transcriptPath: readOption(args, "--transcript-path")
+  });
+  process.stdout.write(`${JSON.stringify(result.payload, null, 2)}\n`);
+  process.exitCode = result.code;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  await main(process.argv.slice(2));
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-apexlang",
-  "version": "0.2.2",
+  "version": "0.4.0",
   "description": "APEXlang support for pi, powered by Oracle's public APEXlang skill.",
   "type": "module",
   "license": "UPL-1.0",
@@ -22,6 +22,7 @@
     "agent-skill"
   ],
   "files": [
+    "docs",
     "extensions",
     "skills",
     "!extensions/**/__pycache__/**",