semantifold 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +71 -0
- package/build/index.d.ts +54 -0
- package/build/index.d.ts.map +1 -0
- package/build/index.js +45 -0
- package/build/src/backends/identifiers.d.ts +19 -0
- package/build/src/backends/identifiers.d.ts.map +1 -0
- package/build/src/backends/identifiers.js +93 -0
- package/build/src/backends/index.d.ts +35 -0
- package/build/src/backends/index.d.ts.map +1 -0
- package/build/src/backends/index.js +149 -0
- package/build/src/backends/java.d.ts +8 -0
- package/build/src/backends/java.d.ts.map +1 -0
- package/build/src/backends/java.js +172 -0
- package/build/src/backends/javascript.d.ts +8 -0
- package/build/src/backends/javascript.d.ts.map +1 -0
- package/build/src/backends/javascript.js +173 -0
- package/build/src/backends/php.d.ts +8 -0
- package/build/src/backends/php.d.ts.map +1 -0
- package/build/src/backends/php.js +175 -0
- package/build/src/backends/ruby.d.ts +8 -0
- package/build/src/backends/ruby.d.ts.map +1 -0
- package/build/src/backends/ruby.js +155 -0
- package/build/src/backends/scalars.d.ts +15 -0
- package/build/src/backends/scalars.d.ts.map +1 -0
- package/build/src/backends/scalars.js +138 -0
- package/build/src/backends/shared.d.ts +26 -0
- package/build/src/backends/shared.d.ts.map +1 -0
- package/build/src/backends/shared.js +424 -0
- package/build/src/backends/typescript.d.ts +8 -0
- package/build/src/backends/typescript.d.ts.map +1 -0
- package/build/src/backends/typescript.js +160 -0
- package/build/src/backends/writer.d.ts +87 -0
- package/build/src/backends/writer.d.ts.map +1 -0
- package/build/src/backends/writer.js +207 -0
- package/build/src/diagnostic.d.ts +67 -0
- package/build/src/diagnostic.d.ts.map +1 -0
- package/build/src/diagnostic.js +96 -0
- package/build/src/frontends/index.d.ts +14 -0
- package/build/src/frontends/index.d.ts.map +1 -0
- package/build/src/frontends/index.js +36 -0
- package/build/src/frontends/java.d.ts +12 -0
- package/build/src/frontends/java.d.ts.map +1 -0
- package/build/src/frontends/java.js +630 -0
- package/build/src/frontends/javascript-typescript.d.ts +19 -0
- package/build/src/frontends/javascript-typescript.d.ts.map +1 -0
- package/build/src/frontends/javascript-typescript.js +633 -0
- package/build/src/frontends/php.d.ts +12 -0
- package/build/src/frontends/php.d.ts.map +1 -0
- package/build/src/frontends/php.js +523 -0
- package/build/src/frontends/ruby.d.ts +12 -0
- package/build/src/frontends/ruby.d.ts.map +1 -0
- package/build/src/frontends/ruby.js +515 -0
- package/build/src/frontends/scalars.d.ts +19 -0
- package/build/src/frontends/scalars.d.ts.map +1 -0
- package/build/src/frontends/scalars.js +71 -0
- package/build/src/mapping.d.ts +160 -0
- package/build/src/mapping.d.ts.map +1 -0
- package/build/src/mapping.js +1120 -0
- package/build/src/semantic/location.d.ts +61 -0
- package/build/src/semantic/location.d.ts.map +1 -0
- package/build/src/semantic/location.js +142 -0
- package/build/src/semantic/operators.d.ts +16 -0
- package/build/src/semantic/operators.d.ts.map +1 -0
- package/build/src/semantic/operators.js +24 -0
- package/build/src/semantic/provenance.d.ts +74 -0
- package/build/src/semantic/provenance.d.ts.map +1 -0
- package/build/src/semantic/provenance.js +577 -0
- package/build/src/semantic/scalars.d.ts +19 -0
- package/build/src/semantic/scalars.d.ts.map +1 -0
- package/build/src/semantic/scalars.js +40 -0
- package/build/src/semantic/types.d.ts +1003 -0
- package/build/src/semantic/types.d.ts.map +1 -0
- package/build/src/semantic/types.js +279 -0
- package/build/src/semantic/validate.d.ts +60 -0
- package/build/src/semantic/validate.d.ts.map +1 -0
- package/build/src/semantic/validate.js +472 -0
- package/docs/architecture.md +39 -0
- package/docs/coding-standards.md +7 -0
- package/docs/goals.md +7 -0
- package/docs/language-support.md +69 -0
- package/docs/plans/2026-09-02-initial-toolchain.md +54 -0
- package/docs/source-maps.md +134 -0
- package/docs/testing.md +25 -0
- package/index.js +63 -0
- package/package.json +69 -0
- package/src/backends/identifiers.js +101 -0
- package/src/backends/index.js +159 -0
- package/src/backends/java.js +182 -0
- package/src/backends/javascript.js +184 -0
- package/src/backends/php.js +184 -0
- package/src/backends/ruby.js +163 -0
- package/src/backends/scalars.js +120 -0
- package/src/backends/shared.js +456 -0
- package/src/backends/typescript.js +168 -0
- package/src/backends/writer.js +224 -0
- package/src/diagnostic.js +103 -0
- package/src/frontends/index.js +36 -0
- package/src/frontends/java.js +735 -0
- package/src/frontends/javascript-typescript.js +718 -0
- package/src/frontends/php.js +593 -0
- package/src/frontends/ruby.js +622 -0
- package/src/frontends/scalars.js +77 -0
- package/src/mapping.js +1260 -0
- package/src/semantic/location.js +154 -0
- package/src/semantic/operators.js +28 -0
- package/src/semantic/provenance.js +629 -0
- package/src/semantic/scalars.js +44 -0
- package/src/semantic/types.js +319 -0
- package/src/semantic/validate.js +529 -0
- package/tsconfig.json +16 -0
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
Semantifold has three explicit layers.
|
|
4
|
+
|
|
5
|
+
1. Frontend adapters in `src/frontends/` invoke mature language parsers and translate only recognized syntax. Babel owns JavaScript and TypeScript parsing, `php-parser` owns PHP, Prism owns Ruby, and Lezer owns Java. Their AST/CST objects never appear in the public API. Adapters enumerate supported children and reject every other parser node; filtering syntax away is not a normalization strategy. Prism node offsets and comment byte ranges are converted from UTF-8 to JavaScript UTF-16 offsets before semantic location construction and metadata source slicing.
|
|
6
|
+
2. `src/semantic/` defines the shared JSDoc-typed discriminated representation and one-based, filename-bearing UTF-16 source locations. `provenance.js` indexes exact source content, deterministic semantic paths/node IDs, resolved symbol IDs, parser-token subranges, and closed source/derived/synthetic origins without exposing parser objects. Scalar types are exact `integer`, `boolean`, or `string` references. Literal nodes carry JavaScript-safe integer, primitive boolean, or parser-decoded Unicode scalar values. Unary and binary expressions carry a closed semantic `operation` and explicit result `type`, never a source operator symbol. Every function, entry point, consequent, and alternate owns a located `Block` containing ordered `Statement` nodes. Statements are local declarations, assignments, conditionals, explicit returns, or prints; an `IfStatement` has a Boolean condition, a consequent block, and an optional alternate block. Source-spelled `else if`, `elsif`, and `elseif` normalize to an alternate block containing one nested conditional.
|
|
7
|
+
3. Source backends in `src/backends/` validate their semantic capabilities before emitting target syntax. Shared validation owns recursive block shape, reachability and all-path return analysis, recursively complete expressions, exhaustive operation/type signatures, function signatures, lexical binding scopes, mutability, declaration-before-use, and strict condition/return/argument/initializer/assignment types. Target-specific validation adds identifier rules, entry-scaffolding name collisions, Java's signed 32-bit `int` literal and compile-time-known arithmetic range, and target string escaping. Every emitter recursively writes blocks and statements through `SourceWriter`; semantic tokens, target anchors, and synthetic layout are mapped as they are emitted. A backend either returns a complete executable program or raises `UNSUPPORTED_CAPABILITY`; it never silently drops nodes or lets an emitter discover malformed shape through a `TypeError`.
|
|
8
|
+
|
|
9
|
+
`index.js` is the public boundary. `parse({language, filename, source})` dispatches to an adapter. `generate({language, module})` validates and returns the compatibility source string. `generateArtifact(...)` uses the same emission path and returns code plus rich and v3 mappings. `src/mapping.js` owns validated lookup, deterministic serialization, mapping import/composition, and diagnostic remapping.
|
|
10
|
+
|
|
11
|
+
## Round trips
|
|
12
|
+
|
|
13
|
+
The tested semantic round trip is source → semantic module → generated source → equivalent semantic module. It preserves modeled meaning, not original spelling, string quotes/escapes, comments, whitespace, visibility, braces, or formatting. Parsed generated-source locations still refer to generated source, while `SemantifoldMapping` explicitly relates generated ranges back to semantic and original ranges. `number` in JS/TypeScript remains deliberately narrowed to semantic `integer` even though those languages permit fractional and special numeric values; only safe integer literals can enter the representation. String values contain complete Unicode scalar values, never source interpolation or lone UTF-16 surrogates, and each backend escapes that value without concatenating source fragments.
|
|
14
|
+
|
|
15
|
+
Frontend source capabilities and backend target capabilities are intentionally separate. Adapters recognize parser-native syntax and retain a transient parser-neutral operation intent only until shared validation resolves identifier and call types. Successful parsing replaces that intent with one of the closed typed operations below before the public module is returned. Invalid operands point to the offending nested expression; mismatched equality points to the unequal operand; and a non-Boolean condition has its own stable diagnostic.
|
|
16
|
+
|
|
17
|
+
| Result | Closed semantic operations |
|
|
18
|
+
| --- | --- |
|
|
19
|
+
| `integer` | `IntegerAdd`, `IntegerSubtract`, `IntegerMultiply`, `IntegerNegate` |
|
|
20
|
+
| `boolean` from integers | `IntegerEqual`, `IntegerNotEqual`, `IntegerLessThan`, `IntegerLessThanOrEqual`, `IntegerGreaterThan`, `IntegerGreaterThanOrEqual` |
|
|
21
|
+
| `boolean` from booleans | `BooleanNot`, `BooleanAnd`, `BooleanOr`, `BooleanEqual`, `BooleanNotEqual` |
|
|
22
|
+
| `boolean` from strings | `StringEqual`, `StringNotEqual` |
|
|
23
|
+
| `string` | `StringConcat` |
|
|
24
|
+
|
|
25
|
+
Binary operands evaluate left to right. `BooleanAnd` and `BooleanOr` are short-circuit operations. Equality accepts only two operands of the same supported scalar type and means scalar value equality. Semantic integers are mathematical integers with one unsigned zero; IEEE-754 signed zero is not a distinct semantic value. Backends map meaning rather than copying syntax: PHP concatenation is `.`, Java string equality is `.equals`, JavaScript/TypeScript/PHP equality is strict, and other operations use their target's matching native construct. Because supported operations cannot otherwise distinguish signed zero, JavaScript and TypeScript canonically stringify observable scalar output when a module contains integer negation or multiplication. Their frontends unwrap that exact parser-native output scaffold during generated-source reparse, retaining the original operation tree without constant folding. Every emitted operation is conservatively parenthesized; generated-source reparsing reconstructs the same semantic tree.
|
|
26
|
+
|
|
27
|
+
A source-valid semantic name can be rejected for a target reserved word, and a safe semantic integer can be rejected by Java's narrower `int` range. Java validates literal-only integer operation trees with exact mathematical integers without folding the IR, rejecting known results outside signed 32-bit `int`; unknown runtime overflow remains outside the portable guarantee.
|
|
28
|
+
|
|
29
|
+
## Blocks, flow, and bindings
|
|
30
|
+
|
|
31
|
+
Blocks preserve source order, evaluation order, and lexical nesting. Validation walks each statement once in order: an unconditional return makes every following statement in that block unreachable, and an `if` returns on all paths only when both its consequent and present alternate do. A non-void function must end every reachable path in an explicit return; optional alternates and empty blocks are valid only when later control flow remains complete. Entry points may not return a value.
|
|
32
|
+
|
|
33
|
+
Binding identity is resolver-owned and exposed as deterministic module-local string IDs, never parser objects or trusted caller numeric IDs. Each validation/provenance pass creates symbol records for functions, parameters, and locals and resolves identifier reads, assignment targets, and calls through lexical function/branch/entry scopes. Module callables, parameters, and locals share one no-shadowing namespace; repeated names are rejected even across disjoint branches in this initial capability. Nested-block declarations do not escape their block. Initializers are validated before declaration and assignment right-hand expressions before the target update, preserving evaluation order and making self-reference a use-before-declaration error.
|
|
34
|
+
|
|
35
|
+
## Provenance and generated mappings
|
|
36
|
+
|
|
37
|
+
Frontend offsets remain parser-owned. Babel token positions, php-parser nodes/tokens, Prism byte locations/comment carriers, and Lezer syntax nodes provide exact function, parameter, local, type, assignment, literal, call, and operator ranges. The JavaScript/TypeScript adapter indexes Babel tokens once by start offset, using direct identifier lookup and bounded binary-search token lookup instead of rescanning the full token stream per semantic node. Source normalization converts only coordinates; it never normalizes source content. CRLF and lone CR count as line breaks and Ruby byte offsets become UTF-16 offsets.
|
|
38
|
+
|
|
39
|
+
Each semantic node carries a JSON-safe source-provenance association through reorder, insertion, deletion, cloning, and serialization. Generation rebuilds canonical paths and node/symbol identities from current traversal and validates any serialized caller provenance before use; aggregate path/kind coincidence is never used to associate a different node. Shared caller objects receive deterministic occurrence records keyed by current JSON Pointer, so reused parameters, types, expressions, and valid statements do not collapse distinct emitted locations. Malformed caller registries, including empty source filenames, are discarded before canonical source fallback. This permits legacy and multi-source caller IR while preventing duplicate/stale IDs, mismatched source ownership, or out-of-bounds ranges from becoming semantic authority. The source-aware writer makes all output spans explicit, retains every contextual origin range across composition boundaries, and leaves no coarse legacy emitter path. Rich maps are frozen and indexed after validation; generated, original, node, and symbol queries include contextual related provenance without duplicate spans. See [source provenance and mappings](source-maps.md).
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Coding standards
|
|
2
|
+
|
|
3
|
+
Production code is ESM JavaScript with `// @ts-check` and concrete JSDoc contracts. TypeScript runs with `allowJs`, `checkJs`, declaration emission, NodeNext resolution, and strict checking. Parser values may use an opaque external type only at an immediate narrowing boundary; semantic and public values use named concrete types.
|
|
4
|
+
|
|
5
|
+
Formatting follows the sibling Velocious conventions: no semicolons, double-quoted strings, two-space JavaScript indentation, spaces inside named imports, one expression per readable chain line, and descriptive JSDoc including parameter and return descriptions. ESLint uses the flat configuration with recommended JavaScript, JSDoc, JSDoc inline-cast, and JSDoc tag-line rules.
|
|
6
|
+
|
|
7
|
+
Keep adapters, semantic types, and backends separate. Never parse source with regex fallbacks. Unsupported syntax and capabilities must be visible diagnostics. Each spec file has one top-level `describe`; generated build/declaration files are never hand-edited.
|
package/docs/goals.md
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Goals
|
|
2
|
+
|
|
3
|
+
Semantifold's long-term goal is a language-neutral toolkit for understanding, transforming, generating, and eventually executing programs through a shared high-level semantic representation. The representation should make types, control flow, calls, declarations, diagnostics, and source provenance understandable without exposing a language parser's private tree.
|
|
4
|
+
|
|
5
|
+
The current release candidate proves one minimal vertical slice across PHP, Ruby, JavaScript, TypeScript, and Java. It includes versioned source provenance, deterministic semantic node/symbol identity, range-based generated mappings, Source Map v3 interop, composition, and diagnostic remapping across that slice. It does not yet provide a general analyzer, transformation API, interpreter, optimizer, project graph, debugger integration, coverage remapping, or lossless formatter.
|
|
6
|
+
|
|
7
|
+
Roadmap candidates include a versioned full semantic schema, more scalar and compound types, expressions and statements, user-authored transformations, richer backend capability negotiation, multi-file modules, cross-language stack/coverage presentation, and a semantic interpreter. LLVM, Wasm, JVM bytecode, package publication automation, and production TensorBuzz changes are explicitly outside this candidate.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Language support
|
|
2
|
+
|
|
3
|
+
## Implemented subset
|
|
4
|
+
|
|
5
|
+
| Language | Exact frontend scalar types | Generated entry point |
|
|
6
|
+
| --- | --- | --- |
|
|
7
|
+
| PHP | `int`, `bool`, and `string` declarations; local `@var` carriers; optional exact `declare(strict_types=1)` | `echo …, PHP_EOL` |
|
|
8
|
+
| Ruby | immediately associated Prism-recognized `[Integer]`, `[bool]`, and `[String]` function/local comments | `puts …` |
|
|
9
|
+
| JavaScript | JSDoc `number`, `boolean`, and `string` on synchronous, non-generator functions and locals | `console.log(…)` |
|
|
10
|
+
| TypeScript | exact `number`, `boolean`, and `string` keyword annotations on synchronous, non-generator functions and locals | `console.log(…)` |
|
|
11
|
+
| Java | primitive `int`/`boolean` and exact `String` or `java.lang.String` declarations, plus a sequenced `public static void main` | `System.out.println(…)` in `Main` |
|
|
12
|
+
|
|
13
|
+
Across every language, a module contains at least one function with exactly two required parameters and a scalar return. Function bodies, entry points, and conditional branches are ordered lexical blocks. They may contain supported local declarations, plain assignments, nested conditionals, explicit returns, and prints in source order. Conditions require semantic `boolean`. An alternate is optional, and source `else if`/`elsif`/`elseif` normalizes to one nested conditional inside the alternate block. Non-void functions must explicitly return on every reachable path; an optional alternate is therefore accepted only when a later statement completes the fallthrough path. A statement following an unconditional return or an exhaustive returning conditional in the same block is unreachable and rejected. Entry points cannot return values.
|
|
14
|
+
|
|
15
|
+
Frontend conditional forms remain deliberately bounded:
|
|
16
|
+
|
|
17
|
+
- JavaScript and TypeScript accept `if` with a block or one supported statement and normalize either to `Block`; directives, switch, labels, conditional expressions, and unmodeled executable/type-only statements are rejected.
|
|
18
|
+
- PHP accepts braced `if`/`elseif`/`else`; alternative colon syntax, `switch`, `match`, inline HTML, and truthy conditions are rejected.
|
|
19
|
+
- Ruby accepts ordinary `if`/`elsif`/`else` with explicit `return`; modifier conditionals, `unless`, `case`, implicit method returns, and truthy conditions are rejected.
|
|
20
|
+
- Java requires braced ordinary branches while accepting a direct `else if`; switch forms, labels, assertions, synchronized blocks, local classes, and recovered grammar nodes are rejected.
|
|
21
|
+
|
|
22
|
+
Every local is an explicitly typed simple identifier with an initializer. Parameters are immutable. Initializers are checked before their new binding becomes visible; assignment right-hand expressions are checked before the target update. Exact scalar types are preserved. Lexical function, branch, and entry scopes reject duplicate declarations, use before declaration, unresolved reads/writes, assignment to immutable bindings, type mismatch, and all shadowing or repeated local spellings in this initial capability. Module function names participate in the same no-shadowing contract, so no parameter or local may capture a callable name. Destructuring, multiple/uninitialized declarations, inferred types, arbitrary lvalues, compound/update assignment, globals/fields/constants, and parameter reassignment are unsupported.
|
|
23
|
+
|
|
24
|
+
Local source profiles are exact:
|
|
25
|
+
|
|
26
|
+
- JavaScript accepts only immediately associated `/** @type {string} */`-style JSDoc followed by one initialized `let` or `const`; `let` is mutable and `const` immutable.
|
|
27
|
+
- TypeScript accepts one initialized `let` or `const` identifier with an exact scalar annotation.
|
|
28
|
+
- Java accepts one initialized explicit scalar declarator; `final` is immutable and an unmodified local is mutable.
|
|
29
|
+
- Ruby requires an immediately associated `# @type [String]`-style comment whose first metadata token is exactly `@type`. Profile-like misspellings such as `@typex` are malformed metadata, not declarations or ordinary comments. Ruby locals default to mutable. An immutable declaration uses the same contiguous carrier block with a separate exact `# @semantifold-immutable` line.
|
|
30
|
+
- PHP requires an immediately associated exact `/** @var string $name */` carrier. PHP locals default to mutable. An immutable declaration uses a docblock containing both exact tags:
|
|
31
|
+
|
|
32
|
+
```php
|
|
33
|
+
/**
|
|
34
|
+
* @var string $name
|
|
35
|
+
* @semantifold-immutable
|
|
36
|
+
*/
|
|
37
|
+
$name = "value";
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The Ruby/PHP tags are Semantifold source-profile metadata, not native runtime immutability. Their adapters validate the exact tag shape, semantic validation rejects later writes, and generators reproduce the metadata so mutability round-trips.
|
|
41
|
+
|
|
42
|
+
Expressions contain parameter/local identifiers, scalar literals, two-argument unqualified calls, and the following typed operations. Every operation node records its result type, and identifier/call operands are resolved before an operation is selected.
|
|
43
|
+
|
|
44
|
+
| Meaning | Ruby | JavaScript / TypeScript | PHP | Java |
|
|
45
|
+
| --- | --- | --- | --- | --- |
|
|
46
|
+
| Integer add, subtract, multiply | `+`, `-`, `*` | `+`, `-`, `*` | `+`, `-`, `*` | `+`, `-`, `*` |
|
|
47
|
+
| Integer negate | unary `-` | unary `-` | unary `-` | unary `-` |
|
|
48
|
+
| Boolean not, short-circuit and/or | `!`, `&&`, `\|\|` | `!`, `&&`, `\|\|` | `!`, `&&`, `\|\|` | `!`, `&&`, `\|\|` |
|
|
49
|
+
| Same-type scalar equality/inequality | `==`, `!=` | `===`, `!==` | `===`, `!==` | `==`, `!=` for integer/Boolean; `.equals` and `!…equals` for strings |
|
|
50
|
+
| Integer ordering | `<`, `<=`, `>`, `>=` | `<`, `<=`, `>`, `>=` | `<`, `<=`, `>`, `>=` | `<`, `<=`, `>`, `>=` |
|
|
51
|
+
| String concatenation | `+` | `+` | `.` | `+` |
|
|
52
|
+
|
|
53
|
+
Operands evaluate left to right, and Boolean and/or short-circuit. Conditions require semantic `boolean`; Ruby, PHP, and JavaScript truthiness is not normalized. Equality is scalar value equality only, requires matching operand types, and never coerces. Java reference `==`/`!=` on strings is rejected; generated Java uses `.equals`. Return values, call arguments, initializers, assignments, and explicit operation result types must exactly match their declared scalar types. Generated operations are parenthesized so reparsing preserves the expression tree rather than relying on coincident target precedence.
|
|
54
|
+
|
|
55
|
+
Integer literals are restricted to JavaScript's safe integer range, `Number.MIN_SAFE_INTEGER` through `Number.MAX_SAFE_INTEGER`, and semantic integers have one unsigned mathematical zero. JavaScript and TypeScript can form IEEE-754 `-0` through negation or multiplication, so generated modules containing either operation stringify their scalar print argument to render both zero signs as `0`; the parser recognizes and removes only this exact entry-point scaffold during reparse. Supported operations cannot otherwise observe a zero sign, and the semantic expression tree is neither rewritten nor folded. Java generation additionally requires primitive-`int` literals and rejects compile-time-known add/subtract/multiply/negate results outside `-2147483648` through `2147483647`. This validation uses exact arithmetic without replacing the semantic expression tree. Runtime arithmetic overflow for non-literal expressions is not a portable guarantee.
|
|
56
|
+
|
|
57
|
+
String literals normalize to their parser-decoded Unicode scalar value. Original quotes and escapes are not retained. PHP and Ruby interpolation, JavaScript/TypeScript template substitution, Java text blocks, PHP heredoc/nowdoc, invalid source encodings, and lone surrogate values are rejected. Backends use target-specific escaping for quotes, backslashes, controls, Unicode, and interpolation introducers. Generated source reparses to the same value.
|
|
58
|
+
|
|
59
|
+
Ruby accepts exactly two required positional parameters. Optional, rest, post, keyword, keyword-rest, and block parameters remain unsupported. Its three annotation comments must form the contiguous block immediately before their function. PHP rejects every `declare` form other than one optional exact `declare(strict_types=1)`. Java calls remain unqualified and `main` may not contain ignored statements.
|
|
60
|
+
|
|
61
|
+
Backends retain narrow ASCII identifier rules and reject target reserved words for function names, parameter names, local declarations, assignment targets, call targets, and identifier references before emission. Bindings are never renamed. TypeScript rejects `arguments` and `eval` for function, parameter, local-declaration, and assignment-target bindings because generated modules are strict mode. PHP rejects exact `GLOBALS`, `_SERVER`, `_GET`, `_POST`, `_FILES`, `_COOKIE`, `_SESSION`, `_REQUEST`, `_ENV`, and `this` parameter bindings; only exact `GLOBALS` and `this` are also rejected for local declarations and assignment targets. PHP function/call names, lowercase near-collisions, assignable non-`GLOBALS` superglobals, and ordinary predefined variables retain their legal roles. Ruby rejects exact `_1` through `_9` function, parameter, local-declaration, and assignment-target bindings while retaining `_10`; references and callees keep their general target identifier policy. JavaScript and other targets retain their own identifier policies. Names owned by generated scaffolding are role-specific: entry locals cannot be `console` in JavaScript/TypeScript or `args`/`System` in Java; function parameters and locals cannot be the generated print receiver `console` in JavaScript/TypeScript or `System` in Java; and module functions cannot be `console` in JavaScript/TypeScript or `puts` in Ruby. Ruby locals named `puts` and PHP locals named `PHP_EOL` remain valid because the generated print forms do not resolve them as local bindings. Java method names and PHP function names occupy namespaces that do not capture their entry scaffolding. PHP boolean printing differs from the other targets; the cross-language execution contract therefore compares a string result rather than claiming portable boolean display formatting.
|
|
62
|
+
|
|
63
|
+
## Diagnostics and limitations
|
|
64
|
+
|
|
65
|
+
All five frontends preserve exact input content and parser-backed semantic token ranges in module provenance. All 25 input-to-output combinations produce range-based rich maps and Source Map v3 sidecars with UTF-16 coordinates and symbol names. Output code uses LF deterministically. Only JavaScript and TypeScript may opt into inline or external `sourceMappingURL` directives; other targets consume sidecars through Semantifold APIs. Mapping does not imply lossless formatting or support for syntax outside the semantic subset. See [source provenance and mappings](source-maps.md).
|
|
66
|
+
|
|
67
|
+
`UNSUPPORTED_SYNTAX` reports parsed constructs outside the subset with the best parser-provided source location. `MISSING_TYPE` reports missing or unsupported atomic annotations and identifies integer, boolean, and string as the supported scalar set. Parser failures use `PARSE_ERROR`. Semantic resolution/type failures use `DUPLICATE_BINDING`, `UNRESOLVED_BINDING`, `USE_BEFORE_DECLARATION`, `IMMUTABLE_ASSIGNMENT`, and `TYPE_MISMATCH`; flow failures use `NON_BOOLEAN_CONDITION`, `MISSING_RETURN`, `UNREACHABLE_STATEMENT`, `ILLEGAL_RETURN_CONTEXT`, and `UNSUPPORTED_STATEMENT`. Backends validate complete recursive block shape, flow, scalar payloads, binding/type agreement, mutability, identifiers, target bounds, and Java's fixed `Main.java` artifact basename before emission. Missing or structurally invalid blocks, statement arrays/elements, declaration names, mutability, type references, initializers, assignment targets/names, assigned expressions, conditional branches, and call-expression fields in caller-supplied IR use their nearest owning node location and report `UNSUPPORTED_CAPABILITY` rather than leaking a native error or producing partial source. A non-`Main.java` Java basename uses the same located capability diagnostic contract.
|
|
68
|
+
|
|
69
|
+
Boxed or coercive types, inferred public types, TypeScript literal/enum/`any`/`unknown`/union types, PHP aliases/nullable declarations, Ruby `boolish`, Java `Boolean`/`char`, null values, floating point, decimal, bigint, symbols, regex values, byte strings, and interpolation are unsupported. Division, remainder, exponentiation, bitwise/shift/update/compound operations, Ruby/PHP `and`/`or`, loose equality, ternary/nullish forms, optional/safe navigation, ranges, casts/assertions, collection/object equality, operator overloading, implicit coercion, and constant folding are also excluded. There is no truthiness, implicit-return or statement-value model, void call statement (roadmap task 005), optional-value model, async/generator model, qualified calls other than Java string `.equals`, classes in the semantic representation, imports, loops, switches, exceptions, overloads, generics, packages, or source-format preservation. Ruby type comments remain a bounded RBS-style convention rather than a full signature parser, and Java generation retains the fixed `Main.java` contract.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Initial toolchain plan
|
|
2
|
+
|
|
3
|
+
Date: 2026-09-02
|
|
4
|
+
|
|
5
|
+
## Narrow release-candidate scope
|
|
6
|
+
|
|
7
|
+
Build one honest vertical slice: a module containing a typed two-integer function, a comparison with two return branches, integer arithmetic and a call from an entry point that prints the result. PHP, Ruby, JavaScript with JSDoc, TypeScript, and Java fixtures must normalize to the same semantic meaning. Anything outside the explicitly modeled nodes fails with an unsupported-syntax or unsupported-capability diagnostic.
|
|
8
|
+
|
|
9
|
+
## Architecture and files
|
|
10
|
+
|
|
11
|
+
- `src/semantic/`: JSDoc-typed discriminated semantic nodes, validation, and diagnostics. Parser-specific AST values never cross this boundary.
|
|
12
|
+
- `src/frontends/`: one adapter per language plus a registry. Mature parsers provide syntax trees and source locations; adapters normalize only the release-candidate subset.
|
|
13
|
+
- `src/backends/`: one source generator per language plus a registry. Each backend declares and checks its supported semantic capabilities.
|
|
14
|
+
- `index.js`: small public API for parsing, validation, generation, and language discovery.
|
|
15
|
+
- `spec/fixtures/`: equivalent source programs in all five languages.
|
|
16
|
+
- `spec/`: focused frontend-equivalence, diagnostics, backend-generation/execution, public API, and repository-contract specs; each file has one top-level `describe`.
|
|
17
|
+
- Root package/build/lint/container/CI files and project documentation describe the implemented subset separately from roadmap goals.
|
|
18
|
+
|
|
19
|
+
## Strict TDD order
|
|
20
|
+
|
|
21
|
+
1. RED: fixtures parsed through the public API cannot yet produce one location-bearing shared semantic module.
|
|
22
|
+
2. GREEN: implement the semantic model and five parser-backed frontend adapters; REFACTOR shared traversal and diagnostics without exposing parser ASTs.
|
|
23
|
+
3. RED: one shared semantic module cannot yet generate and execute exact-output programs in all five target languages.
|
|
24
|
+
4. GREEN: implement capability-checked source backends and isolated temporary-directory execution tests; REFACTOR shared emitter helpers only where semantics stay obvious.
|
|
25
|
+
5. RED/GREEN: add explicit unsupported-syntax, public API, package, TensorBuzz-only, and development-environment contracts.
|
|
26
|
+
|
|
27
|
+
Exact focused RED commands and their observed failures will be retained for the final handoff.
|
|
28
|
+
|
|
29
|
+
## Validation
|
|
30
|
+
|
|
31
|
+
Run focused specs while iterating, then `npm test`, `npm run lint`, `npm run typecheck`, `npm run build`, `npm audit --audit-level=high`, `npm ls --omit=dev --all`, `npm pack --dry-run --json`, `git diff --check`, and available Dockerfile/Compose static validation. Runtime tests must invoke `php`, `ruby`, `node`, `tsc` plus `node`, and `javac` plus `java`; a missing executable is a test failure, never a skip.
|
|
32
|
+
|
|
33
|
+
## Risks and controls
|
|
34
|
+
|
|
35
|
+
- Parser ASTs differ significantly: keep traversal local to each adapter and cover fixtures plus unsupported nodes.
|
|
36
|
+
- Language type systems do not align perfectly: use the single semantic `integer` type now and document lossy normalization and generation.
|
|
37
|
+
- Source locations have different coordinate conventions: normalize to one-based line/column spans and preserve the originating filename/language.
|
|
38
|
+
- Ruby lacks enforced parameter/return annotations in the chosen subset: accept explicit RBS-style `# @param`/`# @return` comments in the fixture and reject missing declarations.
|
|
39
|
+
- Generated Java needs a class/file contract: emit `Main.java`; other backends use fixed documented filenames.
|
|
40
|
+
- Tool availability can hide false confidence: execute real toolchains and fail with the missing command named in the diagnostic.
|
|
41
|
+
|
|
42
|
+
## Deliberate non-goals
|
|
43
|
+
|
|
44
|
+
No general-purpose parsing, comments/formatting round trips, classes, mutable variables, loops, exceptions, imports/packages, overloads, optimization, interpreter, LLVM, Wasm, JVM bytecode, or publication. Semantic round-trip means equivalent modeled meaning, not reproduction of original source text.
|
|
45
|
+
|
|
46
|
+
## Bounded correction status
|
|
47
|
+
|
|
48
|
+
The 2026-09-02 independent correction pass tightened the frozen candidate without broadening the semantic model:
|
|
49
|
+
|
|
50
|
+
- Frontends now reject non-safe integer literals before construction, JS/TS async or generator flags, Java syntax that would otherwise be filtered, non-local Ruby annotations and non-required parameter forms, and PHP declarations other than optional exact `strict_types=1`.
|
|
51
|
+
- Backends now validate target lexical/reserved identifiers at every semantic name use, Java signed 32-bit integer bounds, exactly one return per branch, exactly one entry print, and two arguments per call before emitters run.
|
|
52
|
+
- The Velocious-compatible inline-JSDoc-cast ESLint plugin is pinned to immutable commit `441b6c5ca335a115a75d9f050134357ddb0a61d1`, with package and lockfile contract coverage.
|
|
53
|
+
|
|
54
|
+
Each accepted review finding has a focused regression spec. The complete original fixture-equivalence and real five-runtime execution coverage remains the release gate.
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# Source provenance and mappings
|
|
2
|
+
|
|
3
|
+
Semantifold keeps one authoritative, language-neutral provenance chain:
|
|
4
|
+
|
|
5
|
+
```text
|
|
6
|
+
generated UTF-16 range -> semantic node/symbol -> original UTF-16 range
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
This works for PHP, Ruby, JavaScript, TypeScript, and Java inputs and outputs. It is not limited to JavaScript `sourceMappingURL` comments. `SemantifoldMapping` v1 is the authoritative range model; every generated artifact also includes an ECMA-426/Source Map v3 projection for existing tooling.
|
|
10
|
+
|
|
11
|
+
## Parsing and semantic provenance
|
|
12
|
+
|
|
13
|
+
`parse()` preserves the existing `location.filename/start/end` shape. Lines and columns are one-based, ranges are half-open, and offsets and columns count UTF-16 code units. CRLF is one line break, a lone CR is one line break, and astral characters occupy two columns and offsets. A CRLF start-of-line coordinate resolves after both code units; the interior offset remains a unique position on the preceding line rather than aliasing the next line. Source content and its original newlines are preserved exactly.
|
|
14
|
+
|
|
15
|
+
Parsed modules additionally have a JSON-safe `provenance` value:
|
|
16
|
+
|
|
17
|
+
```js
|
|
18
|
+
const module = parse({
|
|
19
|
+
filename: "choose.ts",
|
|
20
|
+
language: "typescript",
|
|
21
|
+
source
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
module.provenance // {
|
|
25
|
+
// schema: "SemantifoldProvenance",
|
|
26
|
+
// version: 1,
|
|
27
|
+
// coordinateSystem: "utf16",
|
|
28
|
+
// sources: [{id: "source:0", filename: "choose.ts", content: source, language: "typescript"}],
|
|
29
|
+
// nodes: [{id: "node:0", path: "", kind: "Module", origin, ranges: {}}],
|
|
30
|
+
// symbols: [{id: "symbol:0", kind: "function", declarationNodeId: "node:1", ...}]
|
|
31
|
+
// }
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Node IDs follow deterministic semantic occurrence order, node paths are JSON Pointers, and symbol IDs identify validated function, parameter, and local bindings. Reusing one caller-assembled semantic object at multiple paths produces one record and identity per occurrence rather than overwriting earlier paths; generation selects shared signature and local type occurrences by their current path in every backend. Object-only provenance queries remain intentionally unambiguous and require an ID when the object occurs more than once. Declaration records and resolved identifier/call records share a symbol ID. `ranges` contains parser-owned semantic tokens such as `name`, `type`, `callee`, `literal`, and `operator`. Babel identifier ranges use token offsets, including the complete source spelling of escaped identifiers while excluding TypeScript annotations. Parser AST/CST values never enter the public object.
|
|
35
|
+
|
|
36
|
+
Each semantic node also carries its own JSON-safe `sourceProvenance` record. That object association—not an array position, old JSON Pointer, or caller-controlled ID—is the source of provenance after a semantic transformation. Reordering, inserting, deleting, serializing, or cloning nodes therefore moves their source ranges with them. Generation rebuilds current paths and registry-local node/symbol IDs; cloned nodes receive distinct generated identities even when they intentionally share one original origin.
|
|
37
|
+
|
|
38
|
+
Origins are a closed union:
|
|
39
|
+
|
|
40
|
+
```js
|
|
41
|
+
{kind: "source", sourceId: "source:0", location}
|
|
42
|
+
{kind: "derived", origins: [{sourceId: "source:0", location, role: "desugared-from"}]}
|
|
43
|
+
{kind: "synthetic", reason: "Java class scaffolding", relatedOrigins: []}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`primaryLocation(origin)` returns the direct location, the first derived origin, the first related synthetic origin, or `undefined`. `getNodeProvenance(module, nodeOrId)` and `getSymbolProvenance(module, symbolId)` query parsed-module identities.
|
|
47
|
+
|
|
48
|
+
Provenance is mutable JSON because semantic transformations need ordinary data. Generation therefore treats caller metadata as evidence, never authority: it validates node association, ranges, source bounds, coordinate consistency, and that a claimed `sourceId` actually owns its location, then rebuilds all paths and node/symbol IDs. Aggregate records are never reassigned to nodes merely because an old path and kind happen to match. Missing, malformed, duplicate, or stale metadata falls back safely to semantic `location` values; a malformed source registry, including any empty filename, is discarded before fallback rather than copied into a generated mapping. Independently attached node origins and ranges survive only when exact source content can still verify their coordinates, including through explicitly provided sources; otherwise generation uses the node's current location. Legacy caller-authored modules without provenance remain accepted. If caller-authored IR reuses one expression or valid statement object at multiple structural occurrences, generation maps each occurrence through its exact JSON Pointer and assigns distinct canonical node IDs.
|
|
49
|
+
|
|
50
|
+
## Generated artifacts
|
|
51
|
+
|
|
52
|
+
`generate()` remains the compatibility API and returns exactly the same source bytes as before this feature. `generateArtifact()` is additive:
|
|
53
|
+
|
|
54
|
+
```js
|
|
55
|
+
const artifact = generateArtifact({
|
|
56
|
+
filename: "Main.java",
|
|
57
|
+
language: "java",
|
|
58
|
+
module
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
artifact.code
|
|
62
|
+
artifact.filename
|
|
63
|
+
artifact.language
|
|
64
|
+
artifact.mapping // SemantifoldMapping v1
|
|
65
|
+
artifact.sourceMap // encoded Source Map v3 object
|
|
66
|
+
artifact.sourceMapFilename
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Default filenames are `program.php`, `program.rb`, `program.js`, `program.ts`, and `Main.java`. Java artifacts may include directories but must retain the basename `Main.java`, matching the generated public class and making the returned filename directly compilable; any other basename is a located `UNSUPPORTED_CAPABILITY` diagnostic for the Java backend. `sources` may supply exact content for legacy or caller-assembled multi-source IR:
|
|
70
|
+
|
|
71
|
+
```js
|
|
72
|
+
generateArtifact({
|
|
73
|
+
language: "ruby",
|
|
74
|
+
module: assembledModule,
|
|
75
|
+
sources: [
|
|
76
|
+
{filename: "first.ts", language: "typescript", content: firstSource},
|
|
77
|
+
{filename: "second.java", language: "java", content: secondSource}
|
|
78
|
+
]
|
|
79
|
+
})
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
This single-artifact envelope is intentionally reusable as an element in task 010's future ordered artifact set. Generation never writes files.
|
|
83
|
+
|
|
84
|
+
All outputs have deterministic LF newlines. Returned rich mappings are detached from caller-owned semantic locations and provenance before they are deeply frozen and internally indexed after trust-boundary validation. Freezing an artifact mapping therefore never freezes the mutable semantic module used to generate it. The rich map's `generated.content` is the exact returned code and its ordered, non-empty spans cover every UTF-16 code unit without gaps or overlaps:
|
|
85
|
+
|
|
86
|
+
- `exact` maps a generated semantic token to an exact parser token/range.
|
|
87
|
+
- `anchor` maps target syntax such as a keyword or delimiter to the owning semantic origin.
|
|
88
|
+
- `synthetic` marks formatting, comments, entry wrappers, and target scaffolding, optionally with related origins. Writer context expands every direct, derived, or synthetic-related range from its semantic nodes and removes only deterministic duplicates.
|
|
89
|
+
|
|
90
|
+
Every span may carry a canonical `nodeId`, `symbolId`, semantic `role`, and Source Map `name`. Sources retain filenames, exact `sourcesContent` when available, and independent registry identity. Rich mappings are the serialization and interchange contract because v3 represents points rather than ranges and cannot retain semantic roles or the closed derived/synthetic model.
|
|
91
|
+
|
|
92
|
+
## Source Map v3 and directives
|
|
93
|
+
|
|
94
|
+
`artifact.sourceMap` exists for all five targets and is generated with `@jridgewell/gen-mapping`; lookup and import use `@jridgewell/trace-mapping`. Semantifold does not hand-code VLQ or tracing. The v3 map contains deterministic `file`, `sources`, `sourcesContent`, `names`, and `mappings` fields.
|
|
95
|
+
|
|
96
|
+
Directives are opt-in and only valid for JavaScript and TypeScript:
|
|
97
|
+
|
|
98
|
+
```js
|
|
99
|
+
generateArtifact({language: "javascript", module, mapDirective: "external"})
|
|
100
|
+
generateArtifact({language: "typescript", module, mapDirective: "inline"})
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`external` appends `//# sourceMappingURL=<relative-map-url>`. `artifact.sourceMapFilename` remains the map artifact's literal logical filename, while the directive is relative to `artifact.filename`'s directory and URL-encodes each filesystem path segment without encoding structural slashes. Both logical paths resolve against one synthetic artifact root before relativization, so mixed leading-parent depths do not become false common segments; for example, generated `../../dist/out.js` and map `../maps/out.js.map` emit `../maps/out.js.map`. `dist/program.js` still advertises `dist/program.js.map` and emits `sourceMappingURL=program.js.map`; a literal `#` or `?` in the sidecar filename is emitted as `%23` or `%3F`. Names containing any ECMAScript line terminator—LF, CR, U+2028, or U+2029—are rejected before emission. `inline` appends a UTF-8 base64 data URL. The default is `none`. PHP, Ruby, and Java still receive sidecar map objects but never receive non-native map comments. Native Java debugger mapping such as JSR-45 would require class-file/toolchain integration and is not part of v1.
|
|
104
|
+
|
|
105
|
+
The directive itself is an unmapped synthetic rich-map span. `artifact.sourceMap` is computed for the program before that self-referential comment is appended; this is required for a stable inline data URL and is the conventional v3 behavior. Use the rich mapping when the directive span itself matters. Rich registries may distinguish same-named sources by `sourceId`; Source Map v3 identifies sources by filename, so callers should use unique filenames when exporting such an assembled registry to v3.
|
|
106
|
+
|
|
107
|
+
## Queries, serialization, composition, and diagnostics
|
|
108
|
+
|
|
109
|
+
```js
|
|
110
|
+
const original = originalPositionFor(artifact.mapping, {offset: generatedOffset})
|
|
111
|
+
const generated = generatedPositionFor(artifact.mapping, {filename: "choose.ts", offset: originalOffset})
|
|
112
|
+
const unambiguous = generatedPositionFor(artifact.mapping, {sourceId: "source:0", offset: originalOffset})
|
|
113
|
+
const nodeSpans = spansForNode(artifact.mapping, original.nodeId)
|
|
114
|
+
const symbolSpans = spansForSymbol(artifact.mapping, original.symbolId)
|
|
115
|
+
|
|
116
|
+
const json = stringifyMapping(artifact.mapping)
|
|
117
|
+
const restored = parseMapping(json)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Reverse lookup indexes every source range in direct, derived, and synthetic-related origins. Zero-length original locations are indexed as exact points, including valid content-backed Source Map anchors at EOF; they match only their precise offset and do not change ordinary half-open range containment. Node and symbol queries likewise index span-level identities plus identities carried by derived and synthetic related origins. Each query preserves generated-span order and returns a span once even when its identity appears through multiple associations.
|
|
121
|
+
|
|
122
|
+
Rich positions accept zero-based UTF-16 offsets or one-based line/column pairs. Serialization recursively sorts object keys and preserves semantically ordered arrays. Parsing rejects the wrong schema/version, malformed identities, unknown source references, stale source coordinates, non-LF generated content, and span gaps/overlaps.
|
|
123
|
+
|
|
124
|
+
`composeMappings(outer, inner)` composes generated ranges through an intermediate rich mapping. It preserves the inner source registry IDs used by retained node, symbol, and span origins. Every related origin is resolved independently by `sourceId`: origins whose registered source has the inner artifact's filename, exact generated content, and no contradictory known language are traced through every covered inner span, while unrelated origins are retained in deterministic order. Selection scans past matching but untraced origins until it finds an actual inner span. An intermediate EOF point has no containing half-open generated span, so composition preserves that untraced point and retains its intermediate source instead of assigning it to the last span or dropping its registry entry. Derived origins remain derived when at least one original range survives; if their only intermediate range resolves to an unmapped inner synthetic span with no related original range, composition adopts that valid synthetic origin and its stable reason instead of constructing an empty derived origin. Outer synthetic spans remain synthetic, retain their stable reason/context, and translate intermediate context into all related original provenance instead of falsely mapping scaffolding as an anchor. Exact equal-length outer ranges are split at every inner mapping boundary. A strict subrange that cannot carry a correspondingly sliced inner origin is conservatively anchored, as are other ranges without a provable one-to-one subdivision; when such an unsplit range crosses multiple inner spans, its derived origin retains every overlapping original range for forward and reverse lookup. `mappingFromSourceMap(v3, generated)` imports a point map as anchor/synthetic ranges, terminates each segment at its generated line end, explicitly represents unmapped prefixes, newlines, and lines as synthetic, and registers `sourceRoot` inputs by TraceMap's resolved source names. Content-backed anchors at EOF remain zero-length point origins and round-trip through both lookup directions, including for empty sources. Each source referenced by a mapped segment must have exact content in `sourcesContent` or a `generated.sources` override; contentless mapped positions are rejected instead of fabricating offsets, so reverse lookup remains indexable and coordinate-correct. Caller source-content overrides prefer exact V3 source names; a suffix override is accepted only when it identifies one map source, and ambiguous suffixes fail loudly before any registry entry is renamed. `composeSourceMaps(outerV3, innerV3)` composes v3 maps using the maintained mapping libraries. It registers both maps' resolved source names in deterministic input order and pairs `sourcesContent` by raw source index, so `sourceRoot` cannot create unused raw-name sources or duplicate content. Because v3 has filenames but no rich registry IDs or generated content identity, V3 composition traces only when exactly one resolved outer source equals the inner map's `file`; basename or suffix matches are not inferred. The test suite compiles a generated TypeScript artifact with real `tsc`, then verifies both JavaScript -> generated TypeScript -> original Ruby compositions.
|
|
125
|
+
|
|
126
|
+
Validation constructs source line starts once, verifies ranges in linear passes, and creates binary-search/interval indexes for generated and original positions. Immutable mappings produced by Semantifold reuse that validated index for subsequent queries. Mutable caller mappings are deliberately revalidated on every public operation so cache reuse never weakens the trust boundary.
|
|
127
|
+
|
|
128
|
+
`remapLocation(generatedLocation, mapping)` returns the complete lookup result. `remapDiagnostic(diagnostic, mapping)` returns a `SemantifoldDiagnostic` at the original location while preserving the prior range as `generatedLocation` and the original diagnostic as `cause`.
|
|
129
|
+
|
|
130
|
+
## Compatibility and release recommendation
|
|
131
|
+
|
|
132
|
+
The feature is source-compatible and additive: `generate()` bytes are characterized for both existing fixture profiles, legacy semantic modules remain valid, and existing `location` fields are unchanged. Parsed modules now serialize an additional enumerable `provenance` property, so consumers that compare or persist entire module JSON should consciously include or omit it.
|
|
133
|
+
|
|
134
|
+
Semantifold is pre-1.0. The recommended release is a minor bump from `0.1.x` to `0.2.0`, with no npm publication as part of this implementation. A future incompatible provenance or mapping representation must use a new schema version rather than reinterpret version 1.
|
package/docs/testing.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Testing
|
|
2
|
+
|
|
3
|
+
Specs use the released `@velocious/testing@0.0.0` framework and standalone `velocious-test` runner with one top-level `describe` per file. Direct value checks use the framework's `expect` API where it improves clarity; predicate-rich diagnostic assertions retain `node:assert/strict`. Frontend tests load real fixtures through Babel, php-parser, Prism, and Lezer and compare modeled meaning after removing source locations and the separately tested provenance index. They separately assert normalized JSDoc and TypeScript types and verify locations exist.
|
|
4
|
+
|
|
5
|
+
Backend tests start from one parsed semantic module, generate every target, parse each generated program back to equivalent modeled meaning, and work in isolated `mkdtemp` directories. They invoke real commands and assert exact `5\n` output:
|
|
6
|
+
|
|
7
|
+
- `php program.php`
|
|
8
|
+
- `ruby program.rb`
|
|
9
|
+
- `node program.js`
|
|
10
|
+
- local lockfile-installed `tsc program.ts`, then `node program.js`
|
|
11
|
+
- `javac Main.java`, then `java -cp … Main`
|
|
12
|
+
|
|
13
|
+
Cleanup runs in `finally`. A missing tool raises the process error and fails the spec; there are no skip branches or mocked compilers. Diagnostic specs cover unsupported syntax, missing JSDoc types, and backend capability rejection.
|
|
14
|
+
|
|
15
|
+
Mapping specs assert all 25 frontend/backend combinations, exact/anchor/synthetic coverage without gaps, semantic-token roles, deterministic source/node/symbol IDs, exact `sourcesContent`, independently decoded v3 maps, rich/v3 composition, malformed caller metadata repair, legacy modules, caller-assembled multi-source IR, and deterministic JSON round trips. Transformation regressions cover same-kind function reorder, insertion, deletion, cloning, ID regeneration, source ownership, shared parameter, expression, and statement occurrences, and their distinct JSON-pointer node identities. UTF-16 cases cover BMP/astral text plus LF, canonical CRLF, and lone CR; escaped Babel identifiers cover declarations, references, and calls. Imported maps are checked against TraceMap on unmapped prefixes and lines; content-backed EOF anchors, including empty sources, round-trip as exact points, while mapped imports without source content are rejected before offset construction. Composition coverage includes derived intermediate provenance that resolves only to unmapped synthetic content, untraced intermediate EOF points whose registry source must remain available, and ordering where a later range remains traceable. Identity-index coverage checks direct, derived, and synthetic-related node/symbol associations for ordered deduplication. External directive coverage includes mixed parent-relative depths and resolution to the advertised map path. A bounded 50-to-200-function scaling test guards linear validation and indexed lookups; deterministic operation-count coverage guards Babel token indexing without wall-clock thresholds. A real `tsc` map is composed through generated TypeScript to original Ruby source. A separate mapped-artifact acceptance executes PHP, Ruby, Node, TypeScript through real `tsc`/Node, and Java through real `javac`/`java`; unavailable commands fail. SHA-256 characterization checks preserve every existing base/local generated byte across all source languages and targets.
|
|
16
|
+
|
|
17
|
+
Local-binding fixtures cover initialized mutable and immutable declarations, plain assignment, function/branch/entry sequencing, exact Ruby/PHP profile metadata, and source locations in all five frontends. Focused negatives cover every declaration/assignment exclusion, declaration-before-use, unresolved and duplicate bindings, callable-name and lexical no-shadowing, immutable parameters/locals, exact type preservation, and RHS-before-target validation. Ruby coverage rejects profile-like tag misspellings, places multibyte text before later function and local carrier comments, asserts their UTF-16 locations, and reparses generated Ruby with the same offset shift. Backend shape coverage applies missing, null, primitive, array, sparse, and unsupported-object values to recursive blocks, statement elements, declaration/assignment fields, and recursively nested call expressions, proving caller-supplied malformed IR receives the nearest located capability diagnostic before emitter dispatch. Backend runtime coverage regenerates and reparses the same binding/mutability semantics, then executes the result through all five real toolchains with exact `yes\n` output. Target-focused coverage rejects JavaScript/TypeScript `console` and Java `args`/`System` entry captures, JavaScript/TypeScript `console` and Java `System` function parameter/local print-receiver captures, and JavaScript/TypeScript `console` and Ruby `puts` function captures before emission. Real `tsc`, PHP, and Ruby coverage demonstrates their strict or reserved binding failures and requires located target-backend rejection for each proven declaration/assignment role. The PHP probe matrix separately covers automatic-global parameters, the narrower `GLOBALS`/`this` assignment restriction, legal function names, assignable non-`GLOBALS` superglobals, ordinary predefined variables, and exact-case near-collisions; other runtime coverage executes safe JavaScript behavior, Ruby `_10`, and TypeScript near-collisions.
|
|
18
|
+
|
|
19
|
+
Statement-sequencing coverage compares all five parser-native frontends, asserts located `Block` nodes and normalized nested alternates, tests optional fallthrough branches and lexical scope, and requires stable diagnostics for truthy conditions, incomplete returns, unreachable statements, illegal entry returns, and language-specific excluded statements. Separate backend specs corrupt block shape and statement kinds before all five emitters, verify nested generated-source reparses and mappings, preserve legacy bytes for unchanged programs, and execute exact multi-line branch behavior through real PHP, Ruby, Node.js, TypeScript, and Java toolchains.
|
|
20
|
+
|
|
21
|
+
Typed-operation fixtures cover all 18 closed unary/binary operations and explicit result types in every frontend. One semantic module is emitted, reparsed, compiled, and run through real `php`, `ruby`, `node`, local `tsc` plus `node`, and `javac` plus `java`, with exact `typed:operators\n` output. A recursive but side-effect-free right operand proves native short-circuit behavior by remaining unevaluated in every target. Focused negatives cover loose/mixed equality, truthiness, mixed addition, every excluded operator family, casts/assertions, object/collection equality, Java reference equality, malformed Java `.equals`, caller-authored unknown/malformed operations, and Java literal/known-expression bounds. Literal arithmetic remains an operation tree, proving validation does not constant-fold.
|
|
22
|
+
|
|
23
|
+
Focused correction specs prove that unsafe numeric literals never enter the semantic tree; async/generator flags are rejected; Java receivers, unsupported arguments, and extra entry statements cannot disappear; Ruby annotations and parameter forms stay function-local; and PHP accepts only its precise strict-types declaration. Backend specs cover target reserved identifiers, Java's signed 32-bit range, and exact one-return branch cardinality. Repository-contract specs parse package metadata, Compose/TensorBuzz YAML, and the Dockerfile AST; they prove the custom ESLint plugin uses an immutable commit and TensorBuzz is the sole CI topology.
|
|
24
|
+
|
|
25
|
+
Run `npx velocious-test spec/repository-contract.spec.js` for the focused repository contract and `npm test` for framework-native discovery of `.spec.js` files under `spec`. An empty selection is a framework failure and exits nonzero. Run `npm run lint`, `npm run typecheck`, and `npm run build` for source quality and declaration generation. Packaging validation is documented in the root README and `AGENTS.md`.
|
package/index.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {parseSource} from "./src/frontends/index.js"
|
|
4
|
+
import {generateArtifactSource, generateSource} from "./src/backends/index.js"
|
|
5
|
+
|
|
6
|
+
export {SemantifoldDiagnostic} from "./src/diagnostic.js"
|
|
7
|
+
export {getNodeProvenance, getSymbolProvenance} from "./src/semantic/provenance.js"
|
|
8
|
+
export {primaryLocation} from "./src/semantic/provenance.js"
|
|
9
|
+
export {
|
|
10
|
+
composeMappings,
|
|
11
|
+
composeSourceMaps,
|
|
12
|
+
generatedPositionFor,
|
|
13
|
+
mappingFromSourceMap,
|
|
14
|
+
originalPositionFor,
|
|
15
|
+
parseMapping,
|
|
16
|
+
remapDiagnostic,
|
|
17
|
+
remapLocation,
|
|
18
|
+
spansForNode,
|
|
19
|
+
spansForSymbol,
|
|
20
|
+
stringifyMapping,
|
|
21
|
+
toSourceMapV3
|
|
22
|
+
} from "./src/mapping.js"
|
|
23
|
+
|
|
24
|
+
/** @type {readonly import("./src/semantic/types.js").SemanticLanguage[]} */
|
|
25
|
+
export const supportedLanguages = Object.freeze(["php", "ruby", "javascript", "typescript", "java"])
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Parses source into Semantifold's shared semantic representation.
|
|
29
|
+
* @param {object} input - Parse request.
|
|
30
|
+
* @param {string} input.filename - Source filename used in diagnostics.
|
|
31
|
+
* @param {import("./src/semantic/types.js").SemanticLanguage} input.language - Source language.
|
|
32
|
+
* @param {string} input.source - Source text.
|
|
33
|
+
* @returns {import("./src/semantic/types.js").SemanticModule} Semantic module.
|
|
34
|
+
*/
|
|
35
|
+
export function parse(input) {
|
|
36
|
+
return parseSource(input)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Generates target-language source from a shared semantic module.
|
|
41
|
+
* @param {object} input - Generation request.
|
|
42
|
+
* @param {import("./src/semantic/types.js").SemanticLanguage} input.language - Target language.
|
|
43
|
+
* @param {import("./src/semantic/types.js").SemanticModule} input.module - Semantic module.
|
|
44
|
+
* @returns {string} Generated source.
|
|
45
|
+
*/
|
|
46
|
+
export function generate(input) {
|
|
47
|
+
return generateSource(input)
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Generates a reusable output artifact with rich and Source Map v3 mappings.
|
|
52
|
+
* @param {object} input - Generation request.
|
|
53
|
+
* @param {import("./src/semantic/types.js").SemanticLanguage} input.language - Target language.
|
|
54
|
+
* @param {import("./src/semantic/types.js").SemanticModule} input.module - Semantic module.
|
|
55
|
+
* @param {string} [input.filename] - Output filename.
|
|
56
|
+
* @param {"none" | "external" | "inline"} [input.mapDirective] - JavaScript-family map directive.
|
|
57
|
+
* @param {string} [input.sourceMapFilename] - External map filename.
|
|
58
|
+
* @param {{filename: string, content: string, language?: import("./src/semantic/types.js").SemanticLanguage}[]} [input.sources] - Sources for caller-authored modules.
|
|
59
|
+
* @returns {import("./src/semantic/types.js").GeneratedArtifact} Generated artifact.
|
|
60
|
+
*/
|
|
61
|
+
export function generateArtifact(input) {
|
|
62
|
+
return generateArtifactSource(input)
|
|
63
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "semantifold",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Language-neutral semantic code analysis, transformation, generation, and execution toolkit",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "build/index.js",
|
|
7
|
+
"types": "build/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./build/index.d.ts",
|
|
11
|
+
"import": "./build/index.js"
|
|
12
|
+
}
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"build/**",
|
|
16
|
+
"docs/**",
|
|
17
|
+
"index.js",
|
|
18
|
+
"LICENSE",
|
|
19
|
+
"README.md",
|
|
20
|
+
"src/**",
|
|
21
|
+
"tsconfig.json"
|
|
22
|
+
],
|
|
23
|
+
"scripts": {
|
|
24
|
+
"all-checks": "npm run lint && npm test",
|
|
25
|
+
"build": "node scripts/clean-build.js && tsc",
|
|
26
|
+
"eslint": "eslint --max-warnings 0 .",
|
|
27
|
+
"lint": "npm run eslint && npm run typecheck",
|
|
28
|
+
"prepack": "npm run build",
|
|
29
|
+
"test": "velocious-test spec",
|
|
30
|
+
"typecheck": "tsc --noEmit"
|
|
31
|
+
},
|
|
32
|
+
"engines": {
|
|
33
|
+
"node": ">=24"
|
|
34
|
+
},
|
|
35
|
+
"repository": {
|
|
36
|
+
"type": "git",
|
|
37
|
+
"url": "git+https://github.com/kaspernj/semantifold.git"
|
|
38
|
+
},
|
|
39
|
+
"author": "kasper@diestoeckels.de",
|
|
40
|
+
"license": "ISC",
|
|
41
|
+
"bugs": {
|
|
42
|
+
"url": "https://github.com/kaspernj/semantifold/issues"
|
|
43
|
+
},
|
|
44
|
+
"homepage": "https://github.com/kaspernj/semantifold#readme",
|
|
45
|
+
"dependencies": {
|
|
46
|
+
"@babel/parser": "^7.28.4",
|
|
47
|
+
"@babel/types": "^7.28.4",
|
|
48
|
+
"@jridgewell/gen-mapping": "^0.3.13",
|
|
49
|
+
"@jridgewell/trace-mapping": "^0.3.31",
|
|
50
|
+
"@lezer/common": "^1.2.3",
|
|
51
|
+
"@lezer/java": "^1.1.3",
|
|
52
|
+
"@ruby/prism": "^1.9.0",
|
|
53
|
+
"comment-parser": "^1.4.1",
|
|
54
|
+
"php-parser": "^3.7.0"
|
|
55
|
+
},
|
|
56
|
+
"devDependencies": {
|
|
57
|
+
"@eslint/js": "^10.0.1",
|
|
58
|
+
"@types/node": "^24.3.0",
|
|
59
|
+
"@velocious/testing": "0.0.0",
|
|
60
|
+
"dockerfile-ast": "^0.7.1",
|
|
61
|
+
"eslint": "^10.0.1",
|
|
62
|
+
"eslint-plugin-jsdoc": "^64.0.2",
|
|
63
|
+
"eslint-plugin-jsdoc-inline-type-casts": "https://github.com/kaspernj/eslint-plugin-jsdoc-inline-type-casts/archive/441b6c5ca335a115a75d9f050134357ddb0a61d1.tar.gz",
|
|
64
|
+
"eslint-plugin-jsdoc-tag-lines": "^1.0.1",
|
|
65
|
+
"globals": "^17.0.0",
|
|
66
|
+
"typescript": "^7.0.0",
|
|
67
|
+
"yaml": "^2.9.0"
|
|
68
|
+
}
|
|
69
|
+
}
|