semantifold 0.5.0 → 0.7.0

package/node_modules/semantifold-tree-sitter-zig-internal/node_modules/tree-sitter/vendor/tree-sitter/lib/src/wasm_store.h

@@ -0,0 +1,31 @@
+#ifndef TREE_SITTER_WASM_H_
+#define TREE_SITTER_WASM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tree_sitter/api.h"
+#include "./parser.h"
+
+bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *language);
+void ts_wasm_store_reset(TSWasmStore *self);
+bool ts_wasm_store_has_error(const TSWasmStore *self);
+
+bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state);
+bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state);
+
+uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self);
+void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address);
+bool ts_wasm_store_call_scanner_scan(TSWasmStore *self, uint32_t scanner_address, uint32_t valid_tokens_ix);
+uint32_t ts_wasm_store_call_scanner_serialize(TSWasmStore *self, uint32_t scanner_address, char *buffer);
+void ts_wasm_store_call_scanner_deserialize(TSWasmStore *self, uint32_t scanner, const char *buffer, unsigned length);
+
+void ts_wasm_language_retain(const TSLanguage *self);
+void ts_wasm_language_release(const TSLanguage *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_WASM_H_

package/node_modules/semantifold-tree-sitter-zig-internal/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "semantifold-tree-sitter-zig-internal",
+  "version": "0.1.0",
+  "private": true,
+  "description": "Private parser-neutral Zig snapshots using Semantifold's isolated Tree-sitter runtime",
+  "type": "module",
+  "main": "./src/zig.js",
+  "files": [
+    "src/zig.js",
+    "LICENSE",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=24"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "@tree-sitter-grammars/tree-sitter-zig": "1.1.2",
+    "tree-sitter": "0.22.4"
+  }
+}

package/node_modules/semantifold-tree-sitter-zig-internal/src/zig.js

@@ -0,0 +1,134 @@
+// @ts-check
+
+import Parser from "tree-sitter"
+import ZigLanguage from "@tree-sitter-grammars/tree-sitter-zig"
+
+/** @typedef {Readonly<{row: number, column: number}>} CstPosition */
+/** @typedef {Readonly<{field: string | null, node: CstNode}>} CstChild */
+/**
+ * @typedef {Readonly<{
+ *   type: string,
+ *   named: boolean,
+ *   extra: boolean,
+ *   error: boolean,
+ *   missing: boolean,
+ *   hasError: boolean,
+ *   startIndex: number,
+ *   endIndex: number,
+ *   startPosition: CstPosition,
+ *   endPosition: CstPosition,
+ *   children: readonly CstChild[]
+ * }>} CstNode
+ */
+/** @typedef {Readonly<{schema: "semantifold.parser-cst", version: 1, language: "zig", root: CstNode}>} CstSnapshot */
+
+/**
+ * Parses the pinned Zig grammar with its isolated compatible runtime.
+ * @param {string} source - Caller-owned source text.
+ * @returns {CstSnapshot} Recursively frozen parser-neutral CST data.
+ */
+export function parseCst(source) {
+  const parser = new Parser()
+
+  // The grammar's declaration intentionally hides its native language handle as unknown.
+  parser.setLanguage(/** @type {Parser.Language} */ (/** @type {unknown} */ (ZigLanguage)))
+  const tree = parser.parse(source)
+  const convertIndex = createIndexConverter(source, tree.rootNode.endIndex)
+  const lineStarts = createLineStarts(source)
+  const root = snapshotNode(tree.rootNode, convertIndex, lineStarts)
+
+  parser.reset()
+  return Object.freeze({language: "zig", root, schema: "semantifold.parser-cst", version: 1})
+}
+
+/**
+ * @param {import("tree-sitter").SyntaxNode} node
+ * @param {(index: number) => number} convertIndex
+ * @param {readonly number[]} lineStarts
+ * @returns {CstNode}
+ */
+function snapshotNode(node, convertIndex, lineStarts) {
+  /** @type {CstChild[]} */
+  const children = []
+
+  for (let index = 0; index < node.childCount; index += 1) {
+    const child = node.child(index)
+
+    if (!child) throw new Error(`Zig Tree-sitter returned no child at index ${index}`)
+    children.push(Object.freeze({
+      field: node.fieldNameForChild(index) ?? null,
+      node: snapshotNode(child, convertIndex, lineStarts)
+    }))
+  }
+  const startIndex = convertIndex(node.startIndex)
+  const endIndex = convertIndex(node.endIndex)
+
+  return Object.freeze({
+    children: Object.freeze(children),
+    endIndex,
+    endPosition: positionForIndex(endIndex, lineStarts),
+    error: node.isError,
+    extra: node.isExtra,
+    hasError: node.hasError,
+    missing: node.isMissing,
+    named: node.isNamed,
+    startIndex,
+    startPosition: positionForIndex(startIndex, lineStarts),
+    type: node.type
+  })
+}
+
+/**
+ * Normalizes either UTF-8 byte offsets or UTF-16 offsets to UTF-16.
+ * @param {string} source
+ * @param {number} rootEndIndex
+ * @returns {(index: number) => number}
+ */
+function createIndexConverter(source, rootEndIndex) {
+  if (rootEndIndex == source.length) return (index) => index
+  const utf8Length = Buffer.byteLength(source, "utf8")
+
+  if (rootEndIndex != utf8Length) throw new Error(`Unsupported Zig Tree-sitter index extent ${rootEndIndex} for source length ${source.length}`)
+  /** @type {Map<number, number>} */
+  const byteToUtf16 = new Map([[0, 0]])
+  let byteIndex = 0
+
+  for (let utf16Index = 0; utf16Index < source.length;) {
+    const codePoint = source.codePointAt(utf16Index)
+
+    if (codePoint == null) throw new Error(`Cannot read source code point at index ${utf16Index}`)
+    const character = String.fromCodePoint(codePoint)
+
+    byteIndex += Buffer.byteLength(character, "utf8")
+    utf16Index += character.length
+    byteToUtf16.set(byteIndex, utf16Index)
+  }
+  return (index) => {
+    const converted = byteToUtf16.get(index)
+
+    if (converted == null) throw new Error(`Zig Tree-sitter returned a non-boundary byte index ${index}`)
+    return converted
+  }
+}
+
+/** @param {string} source @returns {readonly number[]} */
+function createLineStarts(source) {
+  const starts = [0]
+
+  for (let index = 0; index < source.length; index += 1) if (source[index] == "\n") starts.push(index + 1)
+  return starts
+}
+
+/** @param {number} index @param {readonly number[]} lineStarts @returns {CstPosition} */
+function positionForIndex(index, lineStarts) {
+  let low = 0
+  let high = lineStarts.length
+
+  while (low + 1 < high) {
+    const middle = Math.floor((low + high) / 2)
+
+    if (/** @type {number} */ (lineStarts[middle]) <= index) low = middle
+    else high = middle
+  }
+  return Object.freeze({column: index - /** @type {number} */ (lineStarts[low]), row: low})
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "semantifold",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "Language-neutral semantic code analysis, transformation, generation, and execution toolkit",
   "type": "module",
   "main": "build/index.js",
@@ -21,17 +21,18 @@
     "tsconfig.json"
   ],
   "workspaces": [
-    "packages/tree-sitter-legacy"
+    "packages/tree-sitter-legacy",
+    "packages/tree-sitter-zig"
   ],
   "scripts": {
     "all-checks": "npm run lint && npm test",
-    "build": "node scripts/clean-build.js && tsc && npm run build --workspace=semantifold-tree-sitter-legacy-workspace",
+    "build": "node scripts/clean-build.js && tsc && npm run build --workspace=semantifold-tree-sitter-legacy-workspace && npm run build --workspace=semantifold-tree-sitter-zig-workspace",
     "eslint": "eslint --max-warnings 0 .",
     "lint": "npm run verify:legacy-runtime && npm run eslint && npm run typecheck",
     "prepack": "npm run verify:legacy-runtime && npm run build",
-    "pretest": "npm run verify:legacy-runtime && npm run build --workspace=semantifold-tree-sitter-legacy-workspace",
+    "pretest": "npm run verify:legacy-runtime && npm run build --workspace=semantifold-tree-sitter-legacy-workspace && npm run build --workspace=semantifold-tree-sitter-zig-workspace",
     "test": "velocious-test spec",
-    "typecheck": "tsc --noEmit && npm run typecheck --workspace=semantifold-tree-sitter-legacy-workspace",
+    "typecheck": "tsc --noEmit && npm run typecheck --workspace=semantifold-tree-sitter-legacy-workspace && npm run typecheck --workspace=semantifold-tree-sitter-zig-workspace",
     "verify:legacy-runtime": "node scripts/verify-legacy-runtime.js"
   },
   "engines": {
@@ -58,6 +59,7 @@
     "comment-parser": "^1.4.1",
     "php-parser": "^3.7.0",
     "semantifold-tree-sitter-legacy-internal": "file:packages/tree-sitter-legacy/runtime",
+    "semantifold-tree-sitter-zig-internal": "file:packages/tree-sitter-zig/runtime",
     "tree-sitter": "0.25.1",
     "tree-sitter-c-sharp": "0.23.5",
     "tree-sitter-dart-orchard": "0.7.0",
@@ -68,10 +70,12 @@
   },
   "bundleDependencies": [
     "semantifold-tree-sitter-legacy-internal",
+    "semantifold-tree-sitter-zig-internal",
     "tree-sitter"
   ],
   "acceptDependencies": {
-    "semantifold-tree-sitter-legacy-internal": "0.1.0"
+    "semantifold-tree-sitter-legacy-internal": "0.1.0",
+    "semantifold-tree-sitter-zig-internal": "0.1.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
@@ -84,6 +88,7 @@
     "eslint-plugin-jsdoc-tag-lines": "^1.0.1",
     "globals": "^17.0.0",
     "semantifold-tree-sitter-legacy-workspace": "0.1.0",
+    "semantifold-tree-sitter-zig-workspace": "0.1.0",
     "typescript": "^7.0.0",
     "yaml": "^2.9.0"
   }

package/src/backends/android.js

@@ -17,7 +17,7 @@ const projectRoot = "generated/android-app"
 const generatorVersion = "0.4.0"
 /** @type {Readonly<Set<import("../semantic/types.js").SemanticLanguage>>} */
 const androidSourceLanguages = new Set([
-  "php", "ruby", "javascript", "typescript", "java", "kotlin", "python", "csharp", "go", "c", "cpp", "rust", "swift", "dart"
+  "php", "ruby", "javascript", "typescript", "java", "kotlin", "python", "csharp", "go", "c", "cpp", "rust", "swift", "dart", "zig"
 ])
 const requestFields = new Set(["assets", "configuration", "language", "module", "program", "resources", "role"])
 const configurationFields = new Set([
@@ -74,7 +74,7 @@ const toolchain = Object.freeze({
   commandLineTools: "11076708",
   compileSdk: 35,
   emulator: "35.6.11",
-  gradle: "8.13",
+  gradle: "8.14",
   java: "21.0.8",
   kotlin: "2.2.10",
   kotlinCompilerSha256: "302d1d8e671e5c3207e6ed62ff11fb555462a628e22a1158254dcaaf7e7394bc",
@@ -440,7 +440,11 @@ val semantifoldKotlinHome = providers.environmentVariable("SEMANTIFOLD_KOTLIN_HO
   ?: throw GradleException("SEMANTIFOLD_ANDROID_INFRASTRUCTURE: SEMANTIFOLD_KOTLIN_HOME is required")
 val semantifoldKotlinStdlib = file("$semantifoldKotlinHome/lib/kotlin-stdlib.jar")
 if (!semantifoldKotlinStdlib.isFile) {
-  throw GradleException("SEMANTIFOLD_ANDROID_INFRASTRUCTURE: pinned Kotlin compile library is missing")
+  throw GradleException("SEMANTIFOLD_ANDROID_INFRASTRUCTURE: pinned Kotlin stdlib is missing")
+}
+
+dependencies {
+  androidTestImplementation(files(semantifoldKotlinStdlib))
 }
 val semantifoldAcceptanceRoot = providers.environmentVariable("SEMANTIFOLD_ANDROID_ACCEPTANCE_ROOT").orNull
   ?: throw GradleException("SEMANTIFOLD_ANDROID_INFRASTRUCTURE: SEMANTIFOLD_ANDROID_ACCEPTANCE_ROOT is required")

package/src/backends/dart.js

@@ -2,7 +2,7 @@
 
 import {unsupportedCapability} from "../diagnostic.js"
 import {finalizeMapping, toSourceMapV3} from "../mapping.js"
-import {emitExpression, validateBackendModule} from "./shared.js"
+import {emitExpression, flatDartExpression, validateBackendModule} from "./shared.js"
 import {dartRuntime} from "./dart-runtime.js"
 import {emitScalarType} from "./scalars.js"
 import {SourceWriter} from "./writer.js"
@@ -23,7 +23,7 @@ export function generateDartPackage({filename, mapDirective, module, sourceMapFi
   if (mapDirective !== undefined || sourceMapFilename !== undefined) {
     unsupportedCapability("dart", "source-map filename or directive option", module.location)
   }
-  const unreadLocals = findUnreadLocals(module)
+  const unreadLocals = findUnreadDartLocals(module)
   const writer = new SourceWriter({filename: "bin/program.dart", language: "dart", module, sources})
 
   emitProgram(module, writer, unreadLocals)
@@ -113,12 +113,12 @@ function emitProgram(module, writer, unreadLocals) {
     writer.synthetic(" ", "body spacing", [declaration], [path])
     writer.mapped("{", {mappingKind: "anchor", node: declaration.body, path: `${path}/body`})
     writer.synthetic("\n", "line break", [declaration], [path])
-    emitBlock(writer, declaration.body, "  ", `${path}/body`, unreadLocals)
+    emitDartBlock(writer, declaration.body, "  ", `${path}/body`, unreadLocals)
     writer.mapped("}", {mappingKind: "anchor", node: declaration.body, path: `${path}/body`})
     writer.synthetic("\n", "line break", [declaration], [path])
   }
   writer.synthetic("\nvoid main() {\n", "Dart generated main scaffold", [module.entryPoint], ["/entryPoint"])
-  emitBlock(writer, module.entryPoint.body, "  ", "/entryPoint/body", unreadLocals)
+  emitDartBlock(writer, module.entryPoint.body, "  ", "/entryPoint/body", unreadLocals)
   writer.synthetic("}\n", "Dart generated main scaffold", [module.entryPoint], ["/entryPoint"])
 }
 
@@ -129,11 +129,13 @@ function emitProgram(module, writer, unreadLocals) {
  * @param {string} indent Current indentation.
  * @param {string} path Exact semantic path.
  * @param {Set<import("../semantic/types.js").LocalDeclaration>} unreadLocals Locals requiring an analyzer-only read.
+ * @param {string} [printTarget] Target-owned output function spelling.
+ * @param {boolean} [multilinePrint] Whether target plumbing requires trailing-comma multiline calls.
  * @returns {void}
  */
-function emitBlock(writer, block, indent, path, unreadLocals) {
+export function emitDartBlock(writer, block, indent, path, unreadLocals, printTarget = "print", multilinePrint = false) {
   block.statements.forEach((statement, index) =>
-    emitStatement(writer, statement, indent, `${path}/statements/${index}`, unreadLocals))
+    emitStatement(writer, statement, indent, `${path}/statements/${index}`, unreadLocals, printTarget, multilinePrint))
 }
 
 /**
@@ -143,9 +145,11 @@ function emitBlock(writer, block, indent, path, unreadLocals) {
  * @param {string} indent Current indentation.
  * @param {string} path Exact semantic path.
  * @param {Set<import("../semantic/types.js").LocalDeclaration>} unreadLocals Locals requiring an analyzer-only read.
+ * @param {string} printTarget Target-owned output function spelling.
+ * @param {boolean} multilinePrint Whether target plumbing requires trailing-comma multiline calls.
  * @returns {void}
  */
-function emitStatement(writer, statement, indent, path, unreadLocals) {
+function emitStatement(writer, statement, indent, path, unreadLocals, printTarget, multilinePrint) {
   writer.synthetic(indent, "indentation", [statement], [path])
   if (statement.kind == "LocalDeclaration") {
     if (!statement.mutable) writer.mapped("final", {mappingKind: "anchor", node: statement, path})
@@ -181,7 +185,7 @@ function emitStatement(writer, statement, indent, path, unreadLocals) {
     writer.synthetic("\n", "line break", [statement], [path])
     return
   }
-  if (statement.kind == "IfStatement") return emitIf(writer, statement, indent, path, unreadLocals)
+  if (statement.kind == "IfStatement") return emitIf(writer, statement, indent, path, unreadLocals, printTarget, multilinePrint)
   if (statement.kind == "ReturnStatement") {
     writer.mapped("return", {mappingKind: "anchor", node: statement, path})
     if (statement.expression) {
@@ -193,10 +197,15 @@ function emitStatement(writer, statement, indent, path, unreadLocals) {
     return
   }
   if (statement.kind == "PrintStatement") {
-    writer.mapped("print", {mappingKind: "anchor", node: statement, path})
-    writer.mapped("(", {mappingKind: "anchor", node: statement, path})
+    const expandedPrint = multilinePrint && writer.column - 1 + printTarget.length +
+      flatDartExpression(writer, statement.expression).length + 3 > 80
+
+    if (printTarget == "print") {
+      writer.mapped(printTarget, {mappingKind: "anchor", name: "print", node: statement, path, role: "callee"})
+    } else writer.synthetic(printTarget, "Dart print plumbing", [statement], [path])
+    writer.synthetic(`(${expandedPrint ? `\n${indent}  ` : ""}`, "Dart print plumbing", [statement], [path])
     emitExpression(writer, statement.expression, `${path}/expression`, "dart", identity)
-    writer.mapped(");", {mappingKind: "anchor", node: statement, path})
+    writer.synthetic(expandedPrint ? `,\n${indent});` : ");", "Dart print plumbing", [statement], [path])
     writer.synthetic("\n", "line break", [statement], [path])
     return
   }
@@ -216,23 +225,27 @@ function emitStatement(writer, statement, indent, path, unreadLocals) {
  * @param {string} indent Current indentation.
  * @param {string} path Exact semantic path.
  * @param {Set<import("../semantic/types.js").LocalDeclaration>} unreadLocals Locals requiring an analyzer-only read.
+ * @param {string} printTarget Target-owned output function spelling.
+ * @param {boolean} multilinePrint Whether target plumbing requires trailing-comma multiline calls.
  * @returns {void}
  */
-function emitIf(writer, statement, indent, path, unreadLocals) {
+function emitIf(writer, statement, indent, path, unreadLocals, printTarget, multilinePrint) {
   writer.mapped("if", {mappingKind: "anchor", node: statement, path})
   writer.synthetic(" (", "Dart conditional scaffold", [statement], [path])
   emitExpression(writer, statement.condition, `${path}/condition`, "dart", identity)
   writer.synthetic(") ", "Dart conditional scaffold", [statement], [path])
   writer.mapped("{", {mappingKind: "anchor", node: statement.consequent, path: `${path}/consequent`})
   writer.synthetic("\n", "line break", [statement], [path])
-  emitBlock(writer, statement.consequent, `${indent}  `, `${path}/consequent`, unreadLocals)
+  emitDartBlock(writer, statement.consequent, `${indent}  `, `${path}/consequent`, unreadLocals, printTarget,
+    multilinePrint)
   writer.synthetic(indent, "indentation", [statement], [path])
   writer.mapped("}", {mappingKind: "anchor", node: statement.consequent, path: `${path}/consequent`})
   if (statement.alternate) {
     writer.synthetic(" else ", "Dart alternate scaffold", [statement], [path])
     writer.mapped("{", {mappingKind: "anchor", node: statement.alternate, path: `${path}/alternate`})
     writer.synthetic("\n", "line break", [statement], [path])
-    emitBlock(writer, statement.alternate, `${indent}  `, `${path}/alternate`, unreadLocals)
+    emitDartBlock(writer, statement.alternate, `${indent}  `, `${path}/alternate`, unreadLocals, printTarget,
+      multilinePrint)
     writer.synthetic(indent, "indentation", [statement], [path])
     writer.mapped("}", {mappingKind: "anchor", node: statement.alternate, path: `${path}/alternate`})
   }
@@ -244,7 +257,7 @@ function emitIf(writer, statement, indent, path, unreadLocals) {
  * @param {import("../semantic/types.js").SemanticModule} module Validated semantic module.
  * @returns {Set<import("../semantic/types.js").LocalDeclaration>} Unread declaration identities.
  */
-function findUnreadLocals(module) {
+export function findUnreadDartLocals(module) {
   /** @type {Set<import("../semantic/types.js").LocalDeclaration>} */
   const unread = new Set()