spl-lang 0.1.1 → 0.3.0

package/dist/tools.js

@@ -1,13 +1,12 @@
 import { spawn } from 'node:child_process';
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
 import { Failure } from './errors.js';
 const MAX_TEXT = 200_000;
 const SHELL_TIMEOUT_MS = 120_000;
 const FETCH_TIMEOUT_MS = 30_000;
 const MAX_REDIRECTS = 5;
-/** Characters that let a command chain, redirect or substitute; rejected while any restriction is active. */
-const SHELL_META = /[&|;<>`$()\r\n^%!]/;
 export const TOOL_SPECS = {
     read_file: {
         name: 'read_file',
@@ -86,6 +85,127 @@ export function splitCommand(command) {
         args.push(cur);
     return args;
 }
+function findMatchingParen(str, openIndex) {
+    let depth = 0;
+    for (let i = openIndex; i < str.length; i++) {
+        if (str[i] === '(')
+            depth++;
+        else if (str[i] === ')') {
+            depth--;
+            if (depth === 0)
+                return i;
+        }
+    }
+    return -1;
+}
+/** Splits a command string by chain/pipeline separators (&&, ||, |, ;, &, \n) outside quotes. */
+export function splitChainedCommands(command) {
+    const subcmds = [];
+    let cur = '';
+    let quote;
+    for (let i = 0; i < command.length; i++) {
+        const ch = command[i];
+        if (quote) {
+            if (ch === quote)
+                quote = undefined;
+            cur += ch;
+        }
+        else if (ch === '"' || ch === "'") {
+            quote = ch;
+            cur += ch;
+        }
+        else if (ch === '\n' || ch === ';') {
+            if (cur.trim())
+                subcmds.push(cur.trim());
+            cur = '';
+        }
+        else if (ch === '&' && command[i + 1] === '&') {
+            if (cur.trim())
+                subcmds.push(cur.trim());
+            cur = '';
+            i++;
+        }
+        else if (ch === '|' && command[i + 1] === '|') {
+            if (cur.trim())
+                subcmds.push(cur.trim());
+            cur = '';
+            i++;
+        }
+        else if (ch === '|') {
+            if (cur.trim())
+                subcmds.push(cur.trim());
+            cur = '';
+        }
+        else if (ch === '&') {
+            const prevNonSpace = cur.trimEnd().slice(-1);
+            if (prevNonSpace === '>') {
+                cur += ch;
+            }
+            else {
+                if (cur.trim())
+                    subcmds.push(cur.trim());
+                cur = '';
+            }
+        }
+        else {
+            cur += ch;
+        }
+    }
+    if (quote)
+        throw new Failure('restriction_violation', 'shell command rejected: unterminated quote');
+    if (cur.trim())
+        subcmds.push(cur.trim());
+    return subcmds;
+}
+/** Finds nested $(...) or `...` command substitutions outside single quotes. */
+export function extractSubstitutions(command) {
+    const subs = [];
+    let quote;
+    for (let i = 0; i < command.length; i++) {
+        const ch = command[i];
+        if (quote === "'") {
+            if (ch === "'")
+                quote = undefined;
+        }
+        else if (quote === '"') {
+            if (ch === '"') {
+                quote = undefined;
+            }
+            else if (ch === '`') {
+                const end = command.indexOf('`', i + 1);
+                if (end !== -1) {
+                    subs.push(command.slice(i + 1, end));
+                    i = end;
+                }
+            }
+            else if (ch === '$' && command[i + 1] === '(') {
+                const end = findMatchingParen(command, i + 1);
+                if (end !== -1) {
+                    subs.push(command.slice(i + 2, end));
+                    i = end;
+                }
+            }
+        }
+        else if (ch === "'" || ch === '"') {
+            quote = ch;
+        }
+        else if (ch === '`') {
+            const end = command.indexOf('`', i + 1);
+            if (end !== -1) {
+                subs.push(command.slice(i + 1, end));
+                i = end;
+            }
+        }
+        else if (ch === '$' && command[i + 1] === '(') {
+            const end = findMatchingParen(command, i + 1);
+            if (end !== -1) {
+                subs.push(command.slice(i + 2, end));
+                i = end;
+            }
+        }
+    }
+    return subs;
+}
 const SEARCH_ENDPOINT = 'https://html.duckduckgo.com/html/';
 const MAX_SEARCH_RESULTS = 10;
 const decodeEntities = (s) => s
@@ -121,6 +241,10 @@ export function parseSearchResults(html) {
 }
 const looksLikeUrl = (s) => /^[a-z][a-z0-9+.-]*:\/\//i.test(s);
 const looksLikePath = (s) => /[\\/]/.test(s) || s.startsWith('.') || s.startsWith('~') || /^[a-z]:/i.test(s);
+function isDirSwitch(executable, arg) {
+    return process.platform === 'win32' && executable.toLowerCase() === 'dir'
+        && /^\/(?:-?[bcdlnpqrswx4]|[aot](?::[a-z-]+)?)$/i.test(arg);
+}
 /**
  * Executes the built-in tools. Every call is checked against the active restrictions,
  * whether it comes from the script itself or from the LLM.
@@ -128,11 +252,17 @@ const looksLikePath = (s) => /[\\/]/.test(s) || s.startsWith('.') || s.startsWit
 export class ToolHost {
     restrictions;
     cwd;
-    constructor(restrictions, cwd) {
+    replay;
+    raw;
+    constructor(restrictions, cwd, replay = false, raw = false) {
         this.restrictions = restrictions;
         this.cwd = cwd;
+        this.replay = replay;
+        this.raw = raw;
     }
     async invoke(name, params) {
+        if (this.replay && name !== 'read_file')
+            throw new Failure('restriction_violation', `replay blocks ${name}; only local reads and recorded generation are allowed`);
         switch (name) {
             case 'read_file': return this.readFile(needString(name, params, 'path'));
             case 'write_file': {
@@ -149,7 +279,8 @@ export class ToolHost {
     async readFile(p) {
         const canonical = this.restrictions.checkPath(p, 'read');
         try {
-            return truncate(await fs.readFile(canonical, 'utf8'));
+            const text = await fs.readFile(canonical, 'utf8');
+            return this.raw ? text : truncate(text);
         }
         catch (e) {
             throw new Failure('tool_error', `read_file '${p}': ${e.message}`);
@@ -180,7 +311,16 @@ export class ToolHost {
                 current = this.restrictions.checkUrl(new URL(res.headers.get('location'), current).href);
                 continue;
             }
-            const body = truncate(await res.text());
+            let text;
+            try {
+                text = await res.text();
+            }
+            catch (e) {
+                if (!(e instanceof Error))
+                    throw e;
+                throw new Failure('tool_error', `web_fetch '${current.href}': ${e.message}`);
+            }
+            const body = this.raw ? text : truncate(text);
             if (!res.ok)
                 throw new Failure('tool_error', `web_fetch '${current.href}': HTTP ${res.status}\n${body}`);
             return body;
@@ -209,17 +349,77 @@ export class ToolHost {
         return results.map((r, i) => `${i + 1}. ${r.title}\n   ${r.url}${r.snippet ? `\n   ${r.snippet}` : ''}`).join('\n');
     }
     async shell(command) {
-        if (!this.restrictions.active)
-            return this.run(command, { shell: true });
-        // With any restriction active only plain commands are accepted, so the checks below are exact.
-        if (SHELL_META.test(command)) {
-            throw new Failure('restriction_violation', 'shell command rejected: chaining, redirection and substitution are not allowed inside a restrict block');
+        if (!command || command.trim() === '')
+            throw new Failure('tool_error', 'shell: empty command');
+        if (this.restrictions.active) {
+            try {
+                this.checkCommandRestrictions(command);
+            }
+            catch (e) {
+                if (e instanceof Failure && e.statusCode === 'restriction_violation') {
+                    throw new Failure('restriction_violation', `${e.reason} (shell command: ${JSON.stringify(command)})`);
+                }
+                throw e;
+            }
         }
-        const argv = splitCommand(command);
-        if (argv.length === 0)
+        return this.run(command, { shell: true });
+    }
+    checkCommandRestrictions(command) {
+        const subcmds = splitChainedCommands(command);
+        if (subcmds.length === 0)
             throw new Failure('tool_error', 'shell: empty command');
-        this.restrictions.checkExec(argv[0]);
-        for (const arg of argv.slice(1)) {
+        for (const sub of subcmds) {
+            this.checkSubcommand(sub);
+        }
+        for (const sub of extractSubstitutions(command)) {
+            this.checkCommandRestrictions(sub);
+        }
+    }
+    checkSubcommand(subcmd) {
+        const argv = splitCommand(subcmd);
+        if (argv.length === 0)
+            return;
+        let execIdx = 0;
+        while (execIdx < argv.length && /^[a-zA-Z_][a-zA-Z0-9_]*=/.test(argv[execIdx])) {
+            execIdx++;
+        }
+        if (execIdx >= argv.length)
+            return;
+        const exe = argv[execIdx].replace(/^[()]+/, '').replace(/[()]+$/, '');
+        if (exe)
+            this.restrictions.checkExec(exe);
+        for (let i = execIdx + 1; i < argv.length; i++) {
+            const arg = argv[i];
+            if (arg === '>' || arg === '>>' || arg === '1>' || arg === '2>') {
+                const target = argv[++i];
+                if (target && !target.startsWith('&') && this.restrictions.has('fs')) {
+                    this.restrictions.checkPath(target, 'write');
+                }
+                continue;
+            }
+            if (arg === '<' || arg === '0<') {
+                const target = argv[++i];
+                if (target && this.restrictions.has('fs')) {
+                    this.restrictions.checkPath(target, 'read');
+                }
+                continue;
+            }
+            if (/^[0-9]?>+/.test(arg)) {
+                const target = arg.replace(/^[0-9]?>+/, '');
+                if (target && !target.startsWith('&') && this.restrictions.has('fs')) {
+                    this.restrictions.checkPath(target, 'write');
+                }
+                continue;
+            }
+            if (/^[0-9]?<+/.test(arg)) {
+                const target = arg.replace(/^[0-9]?<+/, '');
+                if (target && this.restrictions.has('fs')) {
+                    this.restrictions.checkPath(target, 'read');
+                }
+                continue;
+            }
+            if (isDirSwitch(exe, arg))
+                continue;
             const value = arg.startsWith('-') && arg.includes('=') ? arg.slice(arg.indexOf('=') + 1) : arg;
             if (looksLikeUrl(value)) {
                 if (this.restrictions.has('net'))
@@ -229,25 +429,20 @@ export class ToolHost {
                 this.restrictions.checkPath(value, 'read');
             }
         }
-        if (argv.some((a) => a.includes('"'))) {
-            throw new Failure('restriction_violation', 'shell command rejected: embedded double quotes are not allowed inside a restrict block');
-        }
-        if (process.platform === 'win32') {
-            // .cmd shims (npm, ...) need a shell on Windows; metacharacters were already rejected.
-            return this.run(argv.map((a) => (/\s/.test(a) || a === '' ? `"${a}"` : a)).join(' '), { shell: true });
-        }
-        return this.run(argv[0], { shell: false, args: argv.slice(1) });
     }
     run(command, opts) {
         return new Promise((resolve, reject) => {
             const child = spawn(command, opts.args ?? [], { cwd: this.cwd, shell: opts.shell, windowsHide: true });
             let out = '';
-            const collect = (d) => {
-                if (out.length < MAX_TEXT * 2)
-                    out += d.toString('utf8');
+            const collect = (text) => {
+                if (this.raw || out.length < MAX_TEXT * 2)
+                    out += text;
             };
-            child.stdout.on('data', collect);
-            child.stderr.on('data', collect);
+            for (const stream of [child.stdout, child.stderr]) {
+                const decoder = new StringDecoder('utf8');
+                stream.on('data', (data) => collect(decoder.write(data)));
+                stream.on('end', () => collect(decoder.end()));
+            }
             const timer = setTimeout(() => child.kill(), SHELL_TIMEOUT_MS);
             child.on('error', (e) => {
                 clearTimeout(timer);
@@ -255,7 +450,7 @@ export class ToolHost {
             });
             child.on('close', (code, signal) => {
                 clearTimeout(timer);
-                const text = truncate(out);
+                const text = this.raw ? out : truncate(out);
                 if (code === 0)
                     resolve(text);
                 else

package/dist/values.d.ts

@@ -17,7 +17,7 @@ export interface UserFunction {
 export interface BuiltinFunction {
     __fn: 'builtin';
     name: string;
-    call: (args: Value[]) => Value;
+    call: (args: Value[]) => Value | Promise<Value>;
 }
 export type FunctionValue = UserFunction | BuiltinFunction;
 export declare const runtimeError: (msg: string) => Failure;

package/docs/syntax.ebnf

@@ -1,97 +1,81 @@
-(* Semantic Prompt Language (SPL) - syntax in EBNF.
-   Notation: ISO-style. { x } = zero or more, [ x ] = optional, ( ) = grouping, | = alternative.
-   Statements are separated by newlines or ';'. Newlines are insignificant inside ( ) and [ ].
-   '#' starts a comment that runs to the end of the line. *)
+(* SPL 0.3. { x } means repetition; [ x ] means optional.
+   Statements end at newline, ';', '}' or EOF. Newlines inside () and [] are ignored.
+   '#' starts a comment. Opening block braces must be on the keyword's line. *)
 
-program        = { directive , NL } , { statement , ( NL | ";" ) } ;
-directive      = "@" , "{" , directive_item , { "," , directive_item } , "}" ;
-directive_item = "model" , "=" , plain_string ;           (* only allowed before the first statement *)
+program        = { directive , separator } , { statement , separator } ;
+separator      = NL | ";" | end_of_block | EOF ;
+directive      = "@" , "{" , "model" , "=" , plain_string , [ "," ] , "}" ;
+block          = "{" , { statement , separator } , "}" ;
 
-block          = "{" , { statement , ( NL | ";" ) } , "}" ;   (* '{' must be on the same line as its keyword; as a statement it is a nested scope *)
-
-statement      = let_stmt | if_stmt | for_stmt | while_stmt | def_stmt | block
-               | "return" , [ expr ]          (* anywhere: ends the function, or the script when at the top level *)
-               | "break" | "continue" | "pass" (* break/continue only inside loops *)
-               | try_stmt | assert_stmt | fail_stmt | success_stmt
-               | constraint_stmt | restrict_stmt | expr ;
-
-(* Every modification of a variable starts with 'let'; a bare  x = 1  is a syntax error.
-   Scoping: 'let NAME = ...' binds in the current scope. A bare block { ... }, a function body and its parameters
-   open a new scope, so 'let' there shadows an outer variable. if/elif/else, for, while, try/except, enforce, forbid
-   and restrict bodies do NOT open a scope: 'let' inside them updates the enclosing variable, which is how results
-   of LLM commands leave an enforce/forbid/restrict block. Reading a name searches the scopes outwards.
-   Compound forms (let x += 1) read the visible value and bind the result in the current scope. *)
-let_stmt       = "let" , target , ( "=" | "+=" | "-=" | "*=" | "/=" | "//=" | "%=" ) , expr ;
-target         = NAME | postfix "." NAME | postfix "[" expr "]" ;
-
-if_stmt         = "if" , expr , block , { NL* "elif" , expr , block } , [ NL* "else" , block ] ;
+statement      = declaration | assignment | if_stmt | for_stmt | while_stmt
+               | def_stmt | try_stmt | guidance_stmt | restrict_stmt | block
+               | "return" , [ expr ] | "success" , [ expr ]
+               | "assert" , expr , [ expr ] | "fail" , expr
+               | "break" | "continue" | "pass" | expr ;
+declaration    = "let" , NAME , "=" , expr ;
+assignment     = target , ( "=" | "+=" | "-=" | "*=" | "/=" | "//=" | "%=" ) , expr ;
+target         = NAME | postfix , "." , NAME | postfix , "[" , expr , "]" ;
+if_stmt        = "if" , expr , block , { NL* , "elif" , expr , block } , [ NL* , "else" , block ] ;
 for_stmt       = "for" , NAME , { "," , NAME } , "in" , expr , block ;
 while_stmt     = "while" , expr , block ;
-def_stmt       = "def" , NAME , "(" , [ NAME , { "," , NAME } ] , ")" , block ;
+def_stmt       = "def" , NAME , "(" , [ NAME , { "," , NAME } , [ "," ] ] , ")" , block ;
 try_stmt       = "try" , block , NL* , "except" , NAME , block ;
+guidance_stmt  = ( "instruct" | "avoid" ) , expr , block ;
+restrict_stmt  = "restrict" , restrict_clause , { "," , restrict_clause } , block ;
+restrict_clause = [ "fs" | "net" | "exec" | "mcp" ] , expr ;
 
-assert_stmt    = "assert" , expr , [ expr ] ;              (* optional message *)
-fail_stmt      = "fail" , expr ;
-success_stmt   = "success" , [ expr ] ;
-
-constraint_stmt = ( "enforce" | "forbid" ) , expr , block ;
-restrict_stmt   = "restrict" , [ "fs" | "net" | "exec" ] , expr , block ;  (* default kind: fs *)
+(* let declares/redeclares in the current function or script. All blocks share that scope.
+   Name assignment requires a declared local; it cannot rebind an enclosing function/script.
+   Functions read lexical enclosing scopes; collection mutations can share state.
+   Loop variables and caught failure names are local bindings.
+   return exits the innermost function, or the script at top level.
+   success exits the script, including when used in a function. *)
 
-(* Expressions, lowest to highest precedence *)
 expr           = or_expr , [ "if" , or_expr , "else" , expr ] ;
 or_expr        = and_expr , { "or" , and_expr } ;
 and_expr       = not_expr , { "and" , not_expr } ;
 not_expr       = "not" , not_expr | comparison ;
-comparison     = sum , { ( "==" | "!=" | "<" | "<=" | ">" | ">=" | "in" | "not" "in" ) , sum } ;
+comparison     = sum , { ( "==" | "!=" | "<" | "<=" | ">" | ">=" | "in" | "not" , "in" ) , sum } ;
 sum            = term , { ( "+" | "-" ) , term } ;
 term           = unary , { ( "*" | "/" | "//" | "%" ) , unary } ;
-unary          = ( "-" | "+" ) , unary | llm_call | power ;
+unary          = ( "-" | "+" ) , unary | "?" , [ modifiers ] , postfix | power ;
 power          = postfix , [ "**" , unary ] ;
 postfix        = primary , { call | index | slice | member } ;
-call           = "(" , [ expr , { "," , expr } ] , ")" ;
+call           = "(" , [ expr , { "," , expr } , [ "," ] ] , ")" ;
 index          = "[" , expr , "]" ;
 slice          = "[" , [ expr ] , ":" , [ expr ] , "]" ;
 member         = "." , NAME ;
-
 primary        = NUMBER | string | "True" | "False" | "None" | NAME
                | "(" , expr , ")" | list | dict
-               | think_call | tool_call ;
+               | ( "think" | "agent" ) , [ modifiers ] , postfix ;
 list           = "[" , [ expr , { "," , expr } , [ "," ] ] , "]" ;
 dict           = "{" , [ expr , ":" , expr , { "," , expr , ":" , expr } , [ "," ] ] , "}" ;
+modifiers      = "{" , [ modifier , { "," , modifier } , [ "," ] ] , "}" ;
+modifier       = ( "len" | "fmt" | "reasoning" | "show" | "schema" | "repairs" | "model" ) , "=" , expr ;
 
-(* LLM commands. '?' is a prefix operator (see 'unary'). The prompt operand is a postfix expression,
-   so  ?'a' + b  is  (?'a') + b  and  -?'n'  is  -(?'n').
-   Every modifier and parameter is optional: ?'q', think 'q' and read_file 'what to do' run with defaults. *)
-llm_call       = "?" , [ ask_mods ] , postfix ;
-think_call     = "think" , [ think_mods ] , postfix ;
-tool_call      = tool_name , [ tool_params ] , postfix ;    (* the last operand holds the instructions *)
-
-ask_mods       = "{" , ask_mod , { "," , ask_mod } , "}" ;
-ask_mod        = "len" , "=" , ( "'word'" | "'sentence'" | "'paragraph'" | "'long'" )
-               | "fmt" , "=" , ( "'text'" | "'object'" ) ;
-think_mods     = "{" , think_mod , { "," , think_mod } , "}" ;
-think_mod      = "level" , "=" , ( "'low'" | "'medium'" | "'high'" )
-               | "show" , "=" , ( "True" | "False" ) ;      (* each modifier at most once *)
-
-tool_name      = "read_file" | "write_file" | "web_fetch" | "web_search" | "shell" ;
-tool_params    = "{" , [ tool_param , { "," , tool_param } ] , "}" ;
-(* When every parameter the tool needs is given, the runtime runs the tool and passes its output to the instructions.
-   When one is missing, the LLM calls the tool itself; the parameters that were given stay fixed. *)
-tool_param     = "path" , "=" , expr       (* read_file, write_file *)
-               | "content" , "=" , expr    (* write_file; the LLM writes the file when omitted *)
-               | "url" , "=" , expr        (* web_fetch *)
-               | "query" , "=" , expr      (* web_search *)
-               | "command" , "=" , expr ;  (* shell *)
+(* Unknown/duplicate modifier names are syntax errors. Values are expressions, validated at runtime:
+   len: word/sentence/paragraph/long (guidance only)
+   fmt: text/json/object; schema implies json unless fmt is explicit
+   reasoning: low/medium/high; think and agent default to medium
+   show: True/False; schema: synchronous strict JSON Schema draft-07 object or boolean
+   repairs: integer 0..5 (additional generation attempts, default 0); agents require 0
+   model: non-empty string; CLI override > per-call model > header > SDK default
+   Generation (? and think) has no tools. agent explicitly enables restricted runtime/MCP tools.
+   Raw tools are ordinary positional functions: read_file(path), write_file(path, content),
+   web_fetch(url), web_search(query), shell(command). They never invoke a model.
+   instruct/avoid are prompt guidance; schema/assert validate data; restrict checks actions. *)
 
-(* Lexical rules *)
-string         = [ "r" ] , ( "'" | '"' | "'''" | '"""' ) , { char | escape | interpolation } , closing_quote ;
-plain_string   = string without interpolation ;
+string         = [ "r" ] , quote , { char | escape | interpolation } , closing_quote ;
+quote          = "'" | '"' | "'''" | '"""' ;
+plain_string   = string_without_interpolation ;
 escape         = "\" , ( "n" | "t" | "r" | "0" | "\" | "'" | '"' | "{" | "}" ) ;
-interpolation  = "{" , expr , "}" ;        (* "{{" and "}}" are literal braces; raw strings (r'...') have no escapes or interpolation *)
+interpolation  = "{" , expr , "}" ;
 NUMBER         = digit+ , [ "." , digit+ ] , [ ( "e" | "E" ) , [ "+" | "-" ] , digit+ ] ;
-NAME           = ( letter | "_" ) , { letter | digit | "_" } ;   (* not a reserved word *)
+NAME           = ( letter | "_" ) , { letter | digit | "_" } ;
 NL             = newline ;
 
-(* Reserved words:
-   let if elif else for in while def return try except assert fail success enforce forbid restrict
-   think break continue pass and or not True False None read_file write_file web_fetch web_search shell *)
+(* "{{"/"}}" are literal braces; raw strings have no interpolation or escapes.
+   Reserved words: let if elif else for in while def return try except assert fail success
+   instruct avoid restrict think agent break continue pass and or not True False None.
+   Removed syntax: enforce/forbid, think{level=...}, tool{param=...} 'instructions',
+   let x += ... and let d.key = ... . *)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "spl-lang",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Semantic Prompt Language: a Python-like language with LLM calls built into the syntax",
   "type": "module",
   "main": "./dist/index.js",
@@ -29,7 +29,8 @@
   ],
   "license": "ISC",
   "dependencies": {
-    "@github/copilot-sdk": "^1.0.22"
+    "@github/copilot-sdk": "^1.0.22",
+    "ajv": "^8.17.1"
   },
   "devDependencies": {
     "@types/node": "^26.6.5",