tree-sitter-delphi13 1.1.1 → 1.2.1

package/README.md

@@ -5,27 +5,37 @@
 
 Tree-sitter grammar pipeline focused on **Delphi 13 (RAD Studio 37.0 / Florence)** and the surrounding modern Delphi ecosystem.
 
-**99.33% pass rate** on a 35,556-file curated Delphi 13 corpus (zero ERROR nodes).
-
-| corpus root | pass rate |
+**v1.2.1: 100.000% pass rate on compilable Delphi 13** (zero ERROR nodes), measured
+on an 11,722-unique-file corpus spanning the RAD Studio 13 RTL/VCL (including
+`System.pas`), DevExpress, Spring4D, EurekaLog, Indy, FireDAC, AsyncPro/Orpheus,
+mORMot-era community code and a full production application. Every failure that
+remains is individually diagnosed, dcc32-cross-checked, and *supposed* to fail
+(see [CORPUS-CEILING-REPORT.md](CORPUS-CEILING-REPORT.md)).
+
+| measurement basis | pass rate |
 |---|---|
-| ORM3 (production application code) | **100.00%** |
-| Spring4D | 99.87% |
-| DevExpress VCL | 99.82% |
-| OmniThreadLibrary | 99.62% |
-| Embarcadero RAD Studio 13 RTL/VCL | 99.24% |
+| orchestrated pipeline (preprocessor → pure), all rows | 99.82% |
+| deduplicated files | 99.74% |
+| deduplicated, valid-Delphi-13-only (excludes broken fixtures, FPC/.NET files) | **100.000%** |
+| own production code (ORM3, drag-lint) | **100.00%** |
 
-The remaining ~0.5% are intentionally broken DUnitX test cases, real vendor source typos, C-language code mistakenly placed in `.pas` files, and a handful of pathologically large auto-generated TypeLib units.
+The 30 files that still fail are intentionally broken test fixtures,
+truncated/corrupted files, FreePascal- or .NET-only sources, and vendor files with
+genuine syntax errors that dcc32 rejects too — a parser that accepted them would be
+wrong. Zero known gaps on valid Delphi 13 remain ([TODO.md](TODO.md) tracks the
+watchlist).
 
 ## Three packages, one repo
 
 | Package | What it does |
 |---|---|
-| **`tree-sitter-delphi13`** *(this directory)* | Master grammar — Delphi-13 syntax, all proven fixes from upstream merged + new extensions. Self-contained "THEN-wins" parser (93.48% on its own). |
+| **`tree-sitter-delphi13`** *(this directory)* | Master grammar — Delphi-13 syntax, all proven fixes from upstream merged + new extensions. Self-contained "THEN-wins" parser (98.6% raw on its own). |
 | **[`tree-sitter-delphi13-pure`](pure/)** | Simpler sub-grammar that drops `pp_*` tokens entirely. Expects preprocessor-resolved source. |
 | **[`delphi13-preprocessor`](preprocessor/)** | Standalone JavaScript library that resolves `{$IFDEF}` / `{$IF}` / `{$DEFINE}` / `{$I X.inc}` directives as a text transformation. |
 
-The full pipeline `preprocessor → pure → AST` is what reaches **99.33%**. Use the master grammar alone if you don't need conditional-compilation awareness; use the pipeline if you do.
+The full pipeline `preprocessor → pure → AST` is what reaches **99.97%** on valid
+Delphi 13. Use the master grammar alone (98.6% raw) if you don't need
+conditional-compilation awareness; use the pipeline if you do.
 
 ## Architecture
 

package/grammar.js

@@ -104,8 +104,18 @@ function enable_if(cond, ...args) {
 // Generate rules for trailing & non-trailing statements
 function statements(trailing) {
 	let rn            = x => trailing ? x + 'Tr' : x
-	let lastStatement = $ => trailing ? optional(tr($,'_statement')) : $._statement;
-	let lastStatement1= $ => trailing ? tr($,'_statement') : $._statement;
+	// Iter 2026-07-16: a goto label may prefix the single-statement BODY of
+	// then/else/do/on (labels already parse at statement-LIST level via
+	// `_statements`): `if Index = 0 then Found: LFNParamStr := ...`
+	// (AsyncPro LFN.pas), `else FoundMismatch: Exit(-1)` (System.pas),
+	// `while true do redo: case ...` (superobject). prec.dynamic(-1): when
+	// the label could equally close an EMPTY body and open a case-arm
+	// (`case x of 1: if c then Foo: ...` — Foo as arm label vs body label),
+	// the arm interpretation must keep winning — that ambiguity is what
+	// cascaded the previous broad-scope attempt.
+	let bodyLabel     = $ => optional(prec.dynamic(-1, $.label));
+	let lastStatement = $ => trailing ? optional(seq(bodyLabel($), tr($,'_statement'))) : seq(bodyLabel($), $._statement);
+	let lastStatement1= $ => trailing ? seq(bodyLabel($), tr($,'_statement')) : seq(bodyLabel($), $._statement);
 	let semicolon     = trailing ? [] : [';'];
 
 	return Object.fromEntries([
@@ -272,6 +282,40 @@ function statements(trailing) {
 	]);
 }
 
+// Shared front of a routine declaration (everything before the first `;`):
+// used by BOTH `_declProc` (strict directive tail — backs defProc, where a
+// body follows) and `_declProcLenient` (separator-form tail with optional
+// final `;` — interface/forward declaration lists only).
+function declProcFront($) {
+	return [
+		...enable_if(fpc, optional($.kGeneric)),
+		optional($.kClass),
+		choice($.kProcedure, $.kFunction, $.kConstructor, $.kDestructor),
+		field('name', $._genericName),
+		field('args', optional($.declArgs)),
+		optional(seq(
+			':',
+			field('type', $.typeref),
+		)),
+		field('assign', optional($.defaultValue)),
+		// Inline calling-convention or procAttribute BEFORE the terminating
+		// ';' — Embarcadero RTL / AzureAPI use lenient forms:
+		//   function DbiInitFn(...): DBIResult stdcall;   (iter 56)
+		//   procedure PreflightBlobRequest(...) overload; (iter 60)
+		// `repeat` allows multiple chained attributes like `... cdecl overload`.
+		repeat(choice(
+			$.kStdcall, $.kCdecl, $.kSafecall, $.kPascal,
+			$.kRegister, $.kWinapi, $.kInline,
+			$.kOverload, $.kVirtual, $.kAbstract, $.kOverride,
+			$.kReintroduce, $.kStatic, $.kDynamic, $.kFinal,
+			// ARC/AUTOREFCOUNT method directive: `class function NewInstance:
+			// TObject unsafe; override;` (System.pas / JsonDataObjects /
+			// FireDAC guard NewInstance with {$IFDEF AUTOREFCOUNT} unsafe).
+			$.kUnsafe
+		)),
+	];
+}
+
 module.exports = grammar({
 	name: "delphi13",
 
@@ -381,6 +425,37 @@ module.exports = grammar({
 		// the two declArg branches ambiguous at the group boundary; let GLR resolve.
 		[$.rttiAttributes],
 		[$.declArg],
+		// Iter 2026-07-16 (implicit-init): the unit tail's `tr($,'block')` arm
+		// forks against `implementation` absorbing the same `begin` into its
+		// `_definitions` (the prec(-1) blockTr recovery in `_definition`).
+		// Reduce-implementation-early must be a live GLR fork; precedence picks
+		// the unit-tail block (prec 0 beats the recovery's -1). This is the
+		// single declarable conflict the old flat-repeat shape could not
+		// express (its fork landed inside `_statementsTr_repeat1`).
+		[$.implementation],
+		// Iter 2026-07-16 (label-as-body): a label prefixing a case-arm's BODY
+		// statement forks against the arm's own jumpLabel slot (`1: Foo: x`).
+		// GLR keeps both; prec.dynamic(-1) on the body label makes the
+		// jumpLabel/arm interpretation win wherever both complete.
+		[$.caseCase],
+		[$.caseCaseTr],
+		// Iter 2026-07-16 (lenient directive tail): inside _declProcLenient a
+		// `;` after an attribute group is either the group SEPARATOR (another
+		// group follows) or the optional FINAL `;` — one further token
+		// decides; GLR forks briefly at the attribute keyword.
+		[$.procAttribute],
+		// ...and the lenient tail's groups fork against declProcFwd's strict
+		// `_procAttributeNoExt` groups while both interpretations are still
+		// live (`function F; stdcall; forward;` — forward only decides later).
+		[$._declProcLenient, $._procAttributeNoExt],
+		// Iter 2026-07-16 (Register as a field name): after `Foo: T;` the
+		// token `Register` is either the previous field's trailing callconv
+		// (`; register;`) or the NEXT field's name (`Register: UINT;`) —
+		// `;` vs `:` one token later decides; GLR forks across the
+		// declField/declFieldNoSemi boundary AND within declField itself
+		// (trailing-callconv slot vs end-of-field).
+		[$.declFieldNoSemi, $.declField],
+		[$.declField],
 		// The following conflict rules are only needed because "public" can be
 		// a visibility or an attribute. *sigh*
 		// TODO: We would probably avoid this by having separate decl* clauses
@@ -404,13 +479,30 @@ module.exports = grammar({
 
 	rules: {
 		root:               $ => optional(choice(
-			$.program,
-			$.library,
-			$.unit,
-			$.package,
+			seq(
+				choice(
+					$.program,
+					$.library,
+					$.unit,
+					$.package,
+				),
+				// Iter 2026-07-16: dcc ignores everything after the final
+				// `end.` (warning W1011 only) — e.g. a stray `uses X;` left
+				// after the closer (mvvm-in-delphi MainScreenForm.pas). One
+				// low-precedence catch-all token; linters can flag the node.
+				optional($.trailingText),
+			),
 			$._definitions // For include files
 		)),
 
+		// Everything from the first non-blank char after the module's final
+		// `end.` to EOF. token(prec(-2)) so real tokens (incl. comments,
+		// which remain ordinary extras after `end.`) always win over it.
+		// NOTE: `(.|\s)*` rather than `[\s\S]*` — tree-sitter's regex engine
+		// compiles the latter to a non-greedy/empty match (token stopped after
+		// one char; observed 2026-07-16).
+		trailingText:       $ => token(prec(-2, /[^\s](.|\s)*/)),
+
 		// HIGH LEVEL ----------------------------------------------------------
 
 		program:            $ => seq(
@@ -443,13 +535,41 @@ module.exports = grammar({
 				$.kLibrary,
 			)),
 			';',
+			// Unit tail, restructured 2026-07-16 (iter: implicit-init). The old
+			// shape was one flat `repeat(choice(interface, implementation,
+			// initialization, finalization))` followed by `kEnd kEndDot`. That
+			// made the Turbo-Pascal implicit initialization block (`begin
+			// <stmts> end.` — still accepted by dcc 13; bdemts.pas, SHDocVw.pas,
+			// System.Win.InternetExplorer.pas, Winapi.OpenGL.PkgHelper.pas,
+			// AsyncPro APFPDENG.pas) unparseable: `optional(blockish)` before
+			// `kEnd` forks against `initialization`'s own statement list and
+			// cascades into the auto-generated `_statementsTr_repeat1` states,
+			// which cannot be named in `conflicts` (attempted + reverted).
+			// The fix: split `initialization`/`finalization` OUT of the flat
+			// repeat into the kEnd tail arm, and give the implicit block its own
+			// mutually-exclusive arm — after `initialization` appears, the block
+			// arm is unreachable, so the fork never exists. The block arm itself
+			// is the same `tr($,'block')` used by `program`/`library` tails, so
+			// the only fork is blockTr-inside-_definitions (the prec(-1)
+			// recovery rule in `_definition`) vs blockTr-as-unit-tail (prec 0),
+			// which GLR resolves by precedence exactly as it already does for
+			// `library`. CST: the implicit block renders as the same `block`
+			// node `program`/`library` produce.
 			repeat(choice(
 				$.interface,
 				$.implementation,
-				$.initialization,
-				$.finalization,
 			)),
-			$.kEnd, $.kEndDot
+			choice(
+				seq(
+					repeat(choice(
+						$.initialization,
+						$.finalization,
+					)),
+					$.kEnd
+				),
+				tr($,'block'),
+			),
+			$.kEndDot
 		),
 
 		// Delphi package file (.dpk). Structure:
@@ -540,6 +660,17 @@ module.exports = grammar({
 		// en/EN not followed by d/D, end immediately followed by ident-char
 		// (so it's NOT the keyword, e.g. 'endif' / 'endloop').
 		asmBody: $ => token(prec(-1, repeat1(choice(
+			// Iter 2026-07-16: consume a whole COMMENT as one chunk, so a bare
+			// `end` *inside* a comment doesn't falsely terminate the asm block.
+			//   add edi,ecx {point EDI to end of destination}   <-- AsyncPro AwFView
+			//   mov eax,1   // end of loop
+			// `{$IFDEF}`-style directives were already safe (the `end[A-Za-z0-9_]`
+			// alternative below covers `endif`), but a comment containing the bare
+			// WORD `end` was not. Brace comments do not nest in Delphi, so
+			// `[^}]*` is exact. These must precede the single-char alternatives:
+			// the lexer takes the longest match, so `{...}` wins over `[^eE]`.
+			/\{[^}]*\}/,
+			/\/\/[^\n]*/,
 			// Identifier not starting with e/E — consume as a unit so an
 			// embedded `end` substring (like in `addend` or `xchg ext`)
 			// doesn't falsely terminate the body.
@@ -792,7 +923,19 @@ module.exports = grammar({
 		typeref:         $ => seq(
 			...enable_if(fpc, field('_dummy', optional($.kSpecialize))),
 			$._typeref,
-			...enable_if(delphi, optional(seq($.kDeprecated, $._expr))),
+			// Trailing declaration hints on the type reference itself. The
+			// kDeprecated+message form predates this note; kPlatform (bare)
+			// added 2026-07-16 for the hint-BEFORE-initializer pattern
+			//   Default8087CW: Word platform = $033F;   (System.pas, x6)
+			// — with the hint absorbed here, declVar's `= expr` defaultValue
+			// follows naturally. A hint-then-value ARM inside declVar itself
+			// was tried first and is a bisect-confirmed generate table bomb
+			// (see TODO.md); this mirrors the existing kDeprecated role-set
+			// instead, which the tables already accommodate.
+			...enable_if(delphi, optional(choice(
+				seq($.kDeprecated, $._expr),
+				$.kPlatform,
+			))),
 		),
 
 		_typeref:        $ => choice(
@@ -850,8 +993,24 @@ module.exports = grammar({
 			$.identifier, $.genericDot, ...enable_if(templates, $.genericTpl)
 		),
 		genericArgs:     $ => delimited1($.genericArg, ';'),
+		// A nested generic instantiation as a genericArg name element — a USE
+		// site reached through the declaration rules: the method resolution
+		// clause `function TFunc<T1, IEnumerable<TResult>>.Invoke = Bind;`
+		// (YADF generic-delegation snippet). Deliberately self-contained
+		// (identifier-only, self-recursive) rather than referencing
+		// genericTpl/_genericName: pulling that mutually-recursive web into
+		// genericArg's item sets explodes table construction (observed: a
+		// 20-minute generate, killed).
+		genericArgTpl:   $ => seq(
+			$.identifier, $.kLt,
+			delimited1(choice($.identifier, $.genericArgTpl)),
+			$.kGt
+		),
 		genericArg:      $ => seq(
-			field('name', delimited1($.identifier)),
+			// Name element: bare identifier (declaration site `TFoo<T; U:
+			// class>`) or a nested instantiation (use site, see genericArgTpl).
+			// `<`-lookahead after the identifier disambiguates.
+			field('name', delimited1(choice($.identifier, $.genericArgTpl))),
 			// Type constraint: `T: BaseType` or one/more of the special
 			// constraint keywords `class` / `record` / `constructor`, comma-
 			// separated. Examples:
@@ -990,7 +1149,12 @@ module.exports = grammar({
 		),
 
 		_declarations:   $ => repeat1(choice(
-			$.declTypes, $.declVars, $.declConsts, $.declProc, $.declProp,
+			$.declTypes, $.declVars, $.declConsts,
+			// Interface declaration lists take the LENIENT directive tail
+			// (final group may omit its `;` — dxCryptoAPI, dxServerModeUtils,
+			// dxGDIPlusAPI); rendered as a plain declProc node.
+			alias($.declProcLenient, $.declProc),
+			$.declProp,
 			alias($.declProcFwd, $.declProc),
 			$.declUses, $.declLabels, $.declExports
 		)),
@@ -1158,6 +1322,13 @@ module.exports = grammar({
 			...enable_if(rtti, optional($.rttiAttributes)),
 			choice($._declProc, $._declOperator),
 		),
+		// Interface/forward flavor: lenient directive tail (see
+		// _declProcLenient). Aliased back to `declProc` at its use site so
+		// the CST is indistinguishable.
+		declProcLenient: $ => seq(
+			...enable_if(rtti, optional($.rttiAttributes)),
+			choice($._declProcLenient, $._declOperator),
+		),
 
 		declVar:         $ => seq(
 			...enable_if(rtti, optional($.rttiAttributes)),
@@ -1191,6 +1362,16 @@ module.exports = grammar({
 				alias($.kDeprecated,   $.identifier),
 				alias($.kExperimental, $.identifier),
 				alias($.kRegister,     $.identifier),
+				// Iter 2026-07-16: same family — `local` is a procAttribute
+				// (FPC directive), so `var X: Integer; Local: Integer;` had the
+				// parser eat `Local` as a trailing directive on X. This is the
+				// L8398/DoSelfTestManifestMerge error f85b412 deferred as "not
+				// the keyword-name family" — it is; it just isn't a hint/callconv.
+				alias($.kLocal,        $.identifier),
+				// Iter 2026-07-16: `dispid` is a property/method directive, so
+				// `var Foo: Integer; DispID: Integer;` hit the same trap.
+				// (System.Win.ObjComAuto L341, Vcl.OleCtrls L2143.)
+				alias($.kDispId,       $.identifier),
 			))),
 			':',
 			// Phase 3b iter 36: declVar type may be a subrange:
@@ -1456,6 +1637,14 @@ module.exports = grammar({
 				// Bare `string`/`ShortString` has no such tail, so it is
 				// unambiguous here.
 				$.kString,
+				// NOTE: `array[...] of T` as a last-field-no-`;` is intentionally
+				// NOT here. Adding declArray fixes SHX/MongoDBCli/ShlObj (+3) but
+				// regresses OoMisc/Z19b5 (-2/3): `array[0..4] of String[1];` (a
+				// short-string element, WITH a trailing `;`) becomes ambiguous with
+				// the no-semi array form and mis-parses. The GLR can't split them
+				// (the `[N]` overlap is lexical). Needs a constrained array element
+				// (exclude declString) to do safely — deferred; not worth the
+				// parser-table cost for ~1 net file. See TODO.
 				prec(-1, $.pp_block),
 			)),
 			field('defaultValue', optional($.defaultValue)),
@@ -1463,7 +1652,18 @@ module.exports = grammar({
 
 		declField:       $ =>  seq(
 			...enable_if(rtti, optional($.rttiAttributes)),
-			field('name', delimited1($.identifier)),
+			// Iter 2026-07-16: `Register` as a FIELD NAME after a prior field
+			// (`Name: PAnsiChar; Register: UINT;` — Winapi.D3D10 shader
+			// reflection records). Without the alias, the trailing-callconv
+			// slot below ate `Register` and choked on the `:`. ONLY kRegister
+			// is aliased — the 2026-07-06 attempt aliased all 7 callconv
+			// keywords into BOTH declField and declFieldNoSemi and blew up
+			// the tables; the narrow form + one-token lookahead (`:` vs `;`)
+			// is all D3D10 needs.
+			field('name', delimited1(choice(
+				$.identifier,
+				alias($.kRegister, $.identifier),
+			))),
 			':',
 			// Iter 2026-07-06: a record/class field type may be a subrange, same
 			// as declVar already allows. `FtrListCount: 0 .. FTRRECMAXCOUNT;`
@@ -1559,35 +1759,40 @@ module.exports = grammar({
 		// Stuff for procedure / function / operator declarations
 
 		_declProc:       $ => seq(
-			...enable_if(fpc, optional($.kGeneric)),
-			optional($.kClass),
-			choice($.kProcedure, $.kFunction, $.kConstructor, $.kDestructor),
-			field('name', $._genericName),
-			field('args', optional($.declArgs)),
-			optional(seq(
-				':',
-				field('type', $.typeref),
-			)),
-			field('assign', optional($.defaultValue)),
-			// Inline calling-convention or procAttribute BEFORE the terminating
-			// ';' — Embarcadero RTL / AzureAPI use lenient forms:
-			//   function DbiInitFn(...): DBIResult stdcall;   (iter 56)
-			//   procedure PreflightBlobRequest(...) overload; (iter 60)
-			// `repeat` allows multiple chained attributes like `... cdecl overload`.
-			repeat(choice(
-				$.kStdcall, $.kCdecl, $.kSafecall, $.kPascal,
-				$.kRegister, $.kWinapi, $.kInline,
-				$.kOverload, $.kVirtual, $.kAbstract, $.kOverride,
-				$.kReintroduce, $.kStatic, $.kDynamic, $.kFinal,
-				// ARC/AUTOREFCOUNT method directive: `class function NewInstance:
-				// TObject unsafe; override;` (System.pas / JsonDataObjects /
-				// FireDAC guard NewInstance with {$IFDEF AUTOREFCOUNT} unsafe).
-				$.kUnsafe
-			)),
+			...declProcFront($),
 			';',
+			// The FINAL directive group may omit its trailing `;` and dcc32
+			// accepts it (VERIFIED 2026-07-16, exit 0):
+			//   `function F(x: Integer): Integer; deprecated 'msg'`
+			//   `function IsEq(...): Boolean; overload`
+			//   `function G: LongWord; stdcall; external 'k32' name 'GetTickCount'`
+			// This STRICT variant does not tolerate that — it backs defProc's
+			// header, where the `;` before the body is mandatory. The lenient
+			// SEPARATOR-form tail lives in `_declProcLenient` (used only via
+			// `_declarations`); a naive `repeat($._procAttributeNoExt)` +
+			// `optional(repeat1(procAttribute))` here would share its whole
+			// prefix and fork inside auto-generated repeat states that cannot
+			// be declared as conflicts (the old KNOWN-GAP note).
 			repeat($._procAttributeNoExt)
 		),
 
+		// Lenient twin of _declProc for FORWARD/interface declaration lists:
+		// directive groups are `;`-SEPARATED with the final `;` optional, so
+		// `function F: Boolean; overload` before another declaration parses.
+		// After each `;` one token of lookahead decides (attribute keyword →
+		// another group, declaration keyword → done) — no shared-prefix fork.
+		// Never used where a body can follow (defProc header stays strict, so
+		// `procedure P; stdcall begin` is still rejected like dcc rejects it).
+		_declProcLenient: $ => seq(
+			...declProcFront($),
+			';',
+			optional(seq(
+				repeat1(field('attribute', $.procAttribute)),
+				repeat(seq(';', repeat1(field('attribute', $.procAttribute)))),
+				optional(';')
+			))
+		),
+
 		_declOperator:   $ => seq(
 			optional($.kClass),
 			$.kOperator,
@@ -1984,7 +2189,10 @@ module.exports = grammar({
 		// so structural tokens stay intact.
 		identifier:        $ => /&{0,2}[a-zA-Z_€-￿][0-9_a-zA-Z€-￿]*/,
 
-	  	_space:            $ => /[\s\r\n\t]+/,
+		// Classic Pascal treats every control char <= #31 as a token
+		// separator (dcc compiles sources with stray bytes like the 0x12 in
+		// DevExpress dxPDFForm.pas), so the whitespace extra does too.
+	  	_space:            $ => /[\s\x00-\x1F]+/,
 		// Single preprocessor directive. The external scanner (src/scanner.c)
 		// handles multi-line `{$IF*}...{$END*}` blocks via the pp_block token,
 		// refusing blocks that wrap structural decls so they parse normally.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tree-sitter-delphi13",
-  "version": "1.1.1",
+  "version": "1.2.1",
   "description": "Tree-sitter grammar focused on Delphi 13 (RAD Studio 37.0)",
   "main": "bindings/node",
   "types": "bindings/node",