ranger-compiler 3.1.1 → 3.4.0

package/dist/lib/core/RgU32.rgr

@@ -0,0 +1,309 @@
+; SPDX-License-Identifier: MIT
+
+; =============================================================================
+; RgU32 — 32-bit unsigned algebra that means the same thing on every target
+; =============================================================================
+; SHA-256, HMAC, clz32, imul, ToInt32 and every checksum stand on this file.
+;
+; It is written the way it is because of measurements, not taste. The same four
+; expressions, compiled by this compiler and run:
+;
+;                          es6            python        go            cpp
+;   2147483647 + 1         2147483648     2147483648    2147483648    -2147483648
+;   bit_shl 1 31           -2147483648    2147483648    2147483648    -2147483648
+;   bit_shl 1 40           256            1099511627776 1099511627776 0
+;   bit_ushr (0-1) 4       268435455      268435455     1152921504606846975
+;
+; Three separate conclusions, each of which rules out an obvious implementation:
+;
+;   1. `int` IS 32-BIT SIGNED ON C++. Plain ARITHMETIC overflows there — the
+;      first row is not a shift, it is `+`. So a u32 cannot be held as an int in
+;      [0, 2^32); that range does not exist on C++. Every intermediate in this
+;      file therefore stays below 2^31, and the u32 is carried as a SIGNED
+;      32-bit BIT PATTERN, which is exactly the range C++ has and exactly what
+;      es6's bitwise operators already produce.
+;
+;   2. `bit_shl` IS NOT PORTABLE. On es6 it is JavaScript's `<<`: the result
+;      wraps to 32 bits and the COUNT is taken mod 32, so `1 << 40` is 256. On
+;      C++ it is undefined past the width and answers 0. On go and python it
+;      grows without bound. Shifts here are therefore bounded to 0..31 and
+;      re-wrapped, and never used to build a value wider than the pattern.
+;
+;   3. `bit_ushr` IS THREE DIFFERENT OPERATORS. 32-bit on es6 and python,
+;      64-bit on go, C++, rust and swift. It is NOT USED ANYWHERE in this file.
+;      `shr` below is built out of a mask, an arithmetic shift and one
+;      re-inserted sign bit instead.
+;
+; What IS safe on all four, and is all this file relies on:
+;   - `bit_and` / `bit_or` / `bit_xor` / `bit_not` on two signed 32-bit patterns.
+;     Wider targets sign-extend both operands identically above bit 31, so the
+;     low 32 bits and the sign agree with the narrow ones.
+;   - `+`, `-`, `*` and `bit_shl`/`bit_shr` on values that stay under 2^31.
+;   - Literals up to 2147483647. Nothing here writes a wider one: 2^32-1 and
+;     2^31 cannot be spelled as int literals on the C++ target at all.
+;
+; The cost is that and/or/xor over full patterns are single operators while
+; `add` and `wrap` split into 16-bit halves. That is a few instructions instead
+; of one, and it is the price of the same answer everywhere. BigIntNum.rgr made
+; the same trade for the same reason ("a product of two limbs is at most 2^30,
+; which is inside a 32-bit int on every target this compiles to").
+;
+; NOT HERE YET: `mul` (i.e. Math.imul). A 32x32 product mod 2^32 needs a 16x16
+; partial product, and 65535 * 65535 is 4294836225 — past what a C++ int holds,
+; so it needs 8-bit limbs and four more partial products. SHA-256, SHA-1, HMAC
+; and the CRC family use only add/xor/and/not/rot/shr, so nothing is blocked;
+; this is written down rather than half-done.
+; =============================================================================
+
+Import "RgNum.rgr"
+
+class RgU32 {
+
+    ; ---- constants that cannot be written as literals -----------------------
+
+    ; 2^31, as a double, because it does not fit a C++ int.
+    static sfn twoPow31D:double () {
+        return 2147483648.0
+    }
+
+    static sfn twoPow32D:double () {
+        return 4294967296.0
+    }
+
+    ; The bit pattern of 0x80000000, i.e. int32 min. Built by subtraction
+    ; because the literal 2147483648 overflows an int on the C++ target.
+    static sfn signBit:int () {
+        def m:int 2147483647
+        return (0 - (m + 1))
+    }
+
+    ; ---- halves -------------------------------------------------------------
+    ; Everything that has to survive a wider target goes through these. Each
+    ; half is in [0, 65535], so every product and sum below stays under 2^31.
+
+    static sfn loHalf:int (v:int) {
+        return (bit_and v 65535)
+    }
+
+    static sfn hiHalf:int (v:int) {
+        return (bit_and (bit_shr v 16) 65535)
+    }
+
+    ; Rebuild a signed 32-bit pattern from two 16-bit halves. The high half's
+    ; top bit becomes the sign, applied by SUBTRACTION rather than by shifting a
+    ; 1 into bit 31 — shifting there is the undefined case on C++.
+    static sfn fromHalves:int (hi:int lo:int) {
+        def h:int (bit_and hi 65535)
+        def l:int (bit_and lo 65535)
+        ; (h & 0x7FFF) << 16 is at most 0x7FFF0000, which fits every target.
+        def r:int (bit_or (bit_shl (bit_and h 32767) 16) l)
+        if ((bit_and h 32768) != 0) {
+            return (r + (RgU32.signBit()))
+        }
+        return r
+    }
+
+    ; Force any int into the signed 32-bit pattern its low 32 bits describe.
+    ; This is ECMAScript's ToInt32 over an integer, and it is what makes a
+    ; go/python result agree with an es6 one.
+    static sfn wrap32:int (v:int) {
+        return (RgU32.fromHalves((RgU32.hiHalf(v)) (RgU32.loHalf(v))))
+    }
+
+    ; ---- logical ------------------------------------------------------------
+    ; Safe as single operators — see the header. Named to avoid the global
+    ; operator namespace, which a static method may not enter.
+
+    static sfn band:int (a:int b:int) {
+        return (bit_and a b)
+    }
+
+    static sfn bor:int (a:int b:int) {
+        return (bit_or a b)
+    }
+
+    static sfn bxor:int (a:int b:int) {
+        return (bit_xor a b)
+    }
+
+    static sfn bnot:int (a:int) {
+        return (RgU32.wrap32((bit_not a)))
+    }
+
+    ; ---- shifts -------------------------------------------------------------
+    ; Counts are taken mod 32, as ECMAScript specifies, and then handled in
+    ; range so no target sees an out-of-width shift.
+
+    static sfn shiftCount:int (n:int) {
+        def c:int (bit_and n 31)
+        return c
+    }
+
+    static sfn shl:int (v:int n:int) {
+        def c:int (RgU32.shiftCount(n))
+        if (c == 0) {
+            return (RgU32.wrap32(v))
+        }
+        ; Shift the halves separately: the wide targets would otherwise carry
+        ; bits past bit 31 that es6 has already dropped.
+        def lo:int (RgU32.loHalf(v))
+        def hi:int (RgU32.hiHalf(v))
+        def full:int 0
+        if (c < 16) {
+            def newLo:int (bit_and (bit_shl lo c) 65535)
+            def carry:int (bit_shr lo (16 - c))
+            def newHi:int (bit_and ((bit_shl hi c) + carry) 65535)
+            full = (RgU32.fromHalves(newHi newLo))
+        } {
+            def d:int (c - 16)
+            def newHi:int (bit_and (bit_shl lo d) 65535)
+            full = (RgU32.fromHalves(newHi 0))
+        }
+        return full
+    }
+
+    ; LOGICAL right shift: zeroes come in at the top. bit_ushr is unusable, so
+    ; the sign bit is dropped, the rest shifted arithmetically, and the dropped
+    ; bit re-inserted at its new position.
+    static sfn shr:int (v:int n:int) {
+        def c:int (RgU32.shiftCount(n))
+        if (c == 0) {
+            return (RgU32.wrap32(v))
+        }
+        def body:int (bit_and v 2147483647)
+        def r:int (bit_shr body c)
+        if (v < 0) {
+            r = (bit_or r (bit_shl 1 (31 - c)))
+        }
+        return r
+    }
+
+    ; ARITHMETIC right shift: the sign bit is replicated. This one `bit_shr`
+    ; already does correctly on every target, once the input is a real pattern.
+    static sfn sar:int (v:int n:int) {
+        def c:int (RgU32.shiftCount(n))
+        if (c == 0) {
+            return (RgU32.wrap32(v))
+        }
+        return (bit_shr (RgU32.wrap32(v)) c)
+    }
+
+    static sfn rotl:int (v:int n:int) {
+        def c:int (RgU32.shiftCount(n))
+        if (c == 0) {
+            return (RgU32.wrap32(v))
+        }
+        def left:int (RgU32.shl(v c))
+        def right:int (RgU32.shr(v (32 - c)))
+        return (bit_or left right)
+    }
+
+    static sfn rotr:int (v:int n:int) {
+        def c:int (RgU32.shiftCount(n))
+        if (c == 0) {
+            return (RgU32.wrap32(v))
+        }
+        def right:int (RgU32.shr(v c))
+        def left:int (RgU32.shl(v (32 - c)))
+        return (bit_or left right)
+    }
+
+    ; ---- arithmetic ---------------------------------------------------------
+
+    ; (a + b) mod 2^32. Through the halves, because `a + b` itself overflows a
+    ; C++ int — which is signed-overflow undefined behaviour there, not merely a
+    ; wrap this code could rely on.
+    static sfn addU:int (a:int b:int) {
+        def lo:int ((RgU32.loHalf(a)) + (RgU32.loHalf(b)))
+        def carry:int (bit_shr lo 16)
+        def hi:int (((RgU32.hiHalf(a)) + (RgU32.hiHalf(b))) + carry)
+        return (RgU32.fromHalves((bit_and hi 65535) (bit_and lo 65535)))
+    }
+
+    static sfn sub:int (a:int b:int) {
+        return (RgU32.addU(a (RgU32.addU((RgU32.bnot(b)) 1))))
+    }
+
+    ; ---- unsigned view ------------------------------------------------------
+    ; The pattern as a NON-NEGATIVE value. It has to be a double: [0, 2^32) is
+    ; not an int range on the C++ target.
+
+    static sfn toUnsignedD:double (v:int) {
+        if (v >= 0) {
+            return (to_double v)
+        }
+        return ((to_double v) + (RgU32.twoPow32D()))
+    }
+
+    ; A non-negative double in [0, 2^32) back to a pattern.
+    static sfn fromUnsignedD:int (d:double) {
+        def two32:double (RgU32.twoPow32D())
+        def x:double d
+        ; Reduce into range without to_int, which saturates at 32 bits on C++.
+        def hiD:double 0.0
+        def loD:double 0.0
+        if (x < 0.0) {
+            x = (x + two32)
+        }
+        if (x >= two32) {
+            x = (x - (two32 * (RgNum.floorD((x / two32)))))
+        }
+        hiD = (RgNum.floorD((x / 65536.0)))
+        loD = (x - (hiD * 65536.0))
+        return (RgU32.fromHalves((to_int hiD) (to_int loD)))
+    }
+
+    ; ---- bit counting -------------------------------------------------------
+
+    ; Leading zeroes in the 32-bit pattern; 32 for zero. This is Math.clz32.
+    static sfn clz:int (v:int) {
+        def p:int (RgU32.wrap32(v))
+        if (p == 0) {
+            return 32
+        }
+        if (p < 0) {
+            return 0
+        }
+        def n:int 0
+        def probe:int 1073741824
+        while (n < 31) {
+            if ((bit_and p probe) != 0) {
+                return (n + 1)
+            }
+            probe = (bit_shr probe 1)
+            n = (n + 1)
+        }
+        return 31
+    }
+
+    static sfn popcount:int (v:int) {
+        def lo:int (RgU32.loHalf(v))
+        def hi:int (RgU32.hiHalf(v))
+        def n:int 0
+        def i:int 0
+        while (i < 16) {
+            if ((bit_and (bit_shr lo i) 1) != 0) {
+                n = (n + 1)
+            }
+            if ((bit_and (bit_shr hi i) 1) != 0) {
+                n = (n + 1)
+            }
+            i = (i + 1)
+        }
+        return n
+    }
+
+    ; ---- byte access --------------------------------------------------------
+    ; Big-endian, because every digest and every length field is.
+
+    static sfn byteAt:int (v:int i:int) {
+        def sh:int (8 * (3 - i))
+        return (bit_and (RgU32.shr(v sh)) 255)
+    }
+
+    static sfn fromBytesBE:int (b0:int b1:int b2:int b3:int) {
+        def hi:int ((bit_shl (bit_and b0 255) 8) + (bit_and b1 255))
+        def lo:int ((bit_shl (bit_and b2 255) 8) + (bit_and b3 255))
+        return (RgU32.fromHalves(hi lo))
+    }
+}

package/dist/lib/ranger-dir.rgr

@@ -1,3 +1,5 @@
+; SPDX-License-Identifier: MIT
+
 
 systemclass  UIElement {
     es6          DOMElement
@@ -118,24 +120,35 @@ class DirListReader {
 (create_polyfill
 "
 
-#ifdef _MSC_VER
+#ifdef _WIN32
     #include <windows.h>
 #else
+    #include <dirent.h>
 #endif
 
 std::vector<std::string> __cpp_list_directory_files(std::string dirName) {
-    WIN32_FIND_DATA data;
     std::vector<std::string> fileList;
+#ifdef _WIN32
+    WIN32_FIND_DATA data;
     std::string query = (dirName + \"\\\\*\");
-    HANDLE hFind = FindFirstFile(query.c_str(), &data);      // DIRECTORY
-
+    HANDLE hFind = FindFirstFile(query.c_str(), &data);
     if ( hFind != INVALID_HANDLE_VALUE ) {
         do {
             std::string name(data.cFileName);
             fileList.push_back( name );
         } while (FindNextFile(hFind, &data));
         FindClose(hFind);
-    }   
+    }
+#else
+    DIR* dir = opendir(dirName.c_str());
+    if (dir != nullptr) {
+        struct dirent* ent;
+        while ((ent = readdir(dir)) != nullptr) {
+            fileList.push_back(std::string(ent->d_name));
+        }
+        closedir(dir);
+    }
+#endif
     return fileList;
 }
 "
@@ -147,6 +160,15 @@ std::vector<std::string> __cpp_list_directory_files(std::string dirName) {
     is_directory cmdFiles:boolean (path:string) {
         templates {
             es6 ( "require('fs').statSync(" (e 1) ").isDirectory()" )
+            cpp ( "r_cpp_is_directory(" (e 1) ")" (imp "<sys/stat.h>") (imp "<string>"
+(polyfill "
+static int r_cpp_is_directory(const std::string& path) {
+    struct stat st;
+    if (stat(path.c_str(), &st) != 0) { return 0; }
+    return S_ISDIR(st.st_mode) ? 1 : 0;
+}
+")
+            )
         }        
     }
     is_file cmdFiles:boolean (path:string) {

package/dist/lib/shell_test.rgr

@@ -0,0 +1,193 @@
+; SPDX-License-Identifier: MIT
+
+; =============================================================================
+; shell_test.rgr -- `lib/Shell.rgr` and the `run_process_result` operator.
+; =============================================================================
+;
+; These tests really do start child processes: the point of this library is that
+; a Ranger program can call another program, and a test that mocked that away
+; would be testing nothing. Everything it runs is `sh`, `printenv` and `ls`,
+; which every POSIX machine has.
+;
+;   npm run shell:test
+;
+; The same file compiled to another target and run there is how the operator's
+; other backends are checked -- the assertions do not know which language they
+; are in.
+; =============================================================================
+
+Import "Shell.rgr"
+
+class ShellTest {
+
+    def passed:int 0
+    def failed:int 0
+
+    ; Argument vectors, built rather than written inline.
+    ;
+    ; A one-element typed array literal handed straight to a call is flattened
+    ; to the bare string on the Rust target (ISSUES.md #84), and ANY array
+    ; literal is lost when the call taking it is immediately dereferenced on
+    ; every target (#85). Every vector in this file goes through one of these
+    ; instead. They also read better than the literal does.
+    sfn one:[string] (a:string) {
+        def v:[string]
+        push v a
+        return v
+    }
+
+    sfn two:[string] (a:string b:string) {
+        def v:[string]
+        push v a
+        push v b
+        return v
+    }
+
+    sfn four:[string] (a:string b:string c:string d:string) {
+        def v:[string]
+        push v a
+        push v b
+        push v c
+        push v d
+        return v
+    }
+
+    fn check:void (name:string cond:boolean) {
+        if cond {
+            passed = passed + 1
+        } {
+            failed = failed + 1
+            print "  FAIL " + name
+        }
+    }
+
+    fn run:void () {
+        def sh:Shell (new Shell)
+
+        ; --- capturing --------------------------------------------------------
+        def hello:ShellResult (sh.capture("sh" (ShellTest.two("-c" "echo hello-shell"))))
+        this.check("a captured command succeeds" (hello.ok()))
+        this.check("and its output comes back trimmed" ((hello.outText()) == "hello-shell"))
+        this.check("and nothing is on stderr" ((hello.errText()) == ""))
+        this.check("and it was not skipped" (hello.skipped == false))
+
+        ; An exit code and a stderr are two different answers and both are
+        ; needed: a tool that failed usually says why on stderr and says
+        ; nothing at all through its status.
+        def bad:ShellResult (sh.capture("sh" (ShellTest.two("-c" "echo to-err 1>&2; exit 7"))))
+        this.check("a failing command reports its code" (bad.code == 7))
+        this.check("and it failed" (bad.failed()))
+        this.check("and stderr came back separately" ((bad.errText()) == "to-err"))
+        this.check("the failure describes itself" ((indexOf (bad.describeFailure()) "exit 7") > 0))
+        this.check("and names the command" ((indexOf (bad.describeFailure()) "to-err") > 0))
+
+        ; --- streaming --------------------------------------------------------
+        ; Nothing to assert about what reached the terminal, but the exit code
+        ; still has to arrive, and that is the half a build driver acts on.
+        def streamed:ShellResult (sh.run("sh" (ShellTest.two("-c" "exit 5"))))
+        this.check("a streamed command reports its code" (streamed.code == 5))
+        this.check("and captures nothing" ((streamed.out == "") && (streamed.err == "")))
+
+        ; --- the conveniences -------------------------------------------------
+        this.check("text answers the trimmed stdout" ((sh.text("sh" (ShellTest.two("-c" "printf one")))) == "one"))
+        this.check("text answers nothing for a failure" ((sh.text("sh" (ShellTest.two("-c" "printf one; exit 1")))) == ""))
+        this.check("succeeds is true for a zero exit" (sh.succeeds("sh" (ShellTest.two("-c" "true")))))
+        this.check("and false otherwise" ((sh.succeeds("sh" (ShellTest.two("-c" "false")))) == false))
+
+        def lines:ShellResult (sh.capture("sh" (ShellTest.two("-c" "printf 'a\nb\nc\n'"))))
+        this.check("output splits into lines" ((array_length (lines.outLines())) == 3))
+        this.check("and the first one is readable on its own" ((lines.outFirstLine()) == "a"))
+        def empty:ShellResult (sh.capture("sh" (ShellTest.two("-c" "true"))))
+        this.check("no output is no lines" ((array_length (empty.outLines())) == 0))
+        this.check("and no first line" ((empty.outFirstLine()) == ""))
+
+        ; --- finding a program ------------------------------------------------
+        this.check("a program on PATH is found" (sh.isInstalled("sh")))
+        this.check("and one that is not is not" ((sh.isInstalled("no-such-program-xyz")) == false))
+        def where@(optional):string (sh.which("sh"))
+        this.check("which answers a path" (!null? where))
+        if (!null? where) {
+            this.check("and it looks like one" ((indexOf (unwrap where) "/") == 0))
+        }
+        def nowhere@(optional):string (sh.which("no-such-program-xyz"))
+        this.check("which answers absent for a program that is not there" (null? nowhere))
+
+        ; --- the working directory --------------------------------------------
+        def inLib:Shell (new Shell)
+        inLib.cwd = "lib"
+        ; Bound rather than chained: a call result that is immediately
+        ; dereferenced loses the array literal in its arguments (ISSUES.md
+        ; #85), and on JavaScript the failure is a runtime type error rather
+        ; than a compile one.
+        def hereArgs:[string] (ShellTest.one("Shell.rgr"))
+        def here:ShellResult (inLib.capture("ls" hereArgs))
+        this.check("cwd is where the command runs" (here.ok()))
+        def elsewhereArgs:[string] (ShellTest.one("package.json"))
+        def elsewhere:ShellResult (inLib.capture("ls" elsewhereArgs))
+        this.check("and a file that is elsewhere is not there" (elsewhere.failed()))
+
+        ; --- the environment --------------------------------------------------
+        def env:Shell (new Shell)
+        env.setEnv("RANGER_SHELL_PROBE" "set-by-ranger")
+        this.check("an environment entry reaches the child" ((env.text("printenv" (ShellTest.one("RANGER_SHELL_PROBE")))) == "set-by-ranger"))
+        ; MERGED, not replaced: a child that lost PATH could not find anything,
+        ; and the failure would look like the program being missing.
+        this.check("and PATH survives it" ((strlen (env.text("printenv" (ShellTest.one("PATH"))))) > 0))
+        env.setEnv("RANGER_SHELL_PROBE" "replaced")
+        this.check("setting one again replaces it" ((env.text("printenv" (ShellTest.one("RANGER_SHELL_PROBE")))) == "replaced"))
+        this.check("rather than adding a second" ((array_length env.env) == 1))
+        this.check("and the logged line shows it" ((indexOf (env.commandLine("cc" (ShellTest.one("-c")))) "RANGER_SHELL_PROBE=replaced") == 0))
+
+        ; --- a program that is not there --------------------------------------
+        ; Every backend answers a non-zero code; which one differs, because the
+        ; targets that go through a shell get the shell's 127.
+        def missing:ShellResult (sh.capture("no-such-program-xyz" (ShellTest.one("x"))))
+        this.check("a missing program is a failure, not a crash" (missing.failed()))
+
+        ; --- quoting ----------------------------------------------------------
+        ; The arguments are passed as a vector, so nothing here is ever re-read
+        ; by a shell. The quoting is for the LOG line -- and a build driver
+        ; whose log cannot be pasted back into a terminal is a build driver
+        ; whose log is decorative.
+        def line:string (sh.commandLine("xcrun" (ShellTest.four("-sdk" "iphonesimulator" "swiftc" "/Users/me/My Projects/a.swift"))))
+        this.check("a plain argument is left alone" ((indexOf line "-sdk iphonesimulator") > 0))
+        this.check("a path with a space is quoted" ((indexOf line "'/Users/me/My Projects/a.swift'") > 0))
+        this.check("an empty argument is visible" ((sh.commandLine("x" (ShellTest.one("")))) == "x ''"))
+
+        ; ...and the vector really is a vector: this argument would be three
+        ; words and a redirection if anything in the chain went through a shell.
+        def literal:ShellResult (sh.capture("printf" (ShellTest.two("%s" "a b > c"))))
+        this.check("an argument is never re-parsed as syntax" ((literal.out) == "a b > c"))
+
+        ; --- the dry run ------------------------------------------------------
+        ; The reason a driver written on this can be tested at all.
+        def dry:Shell (new Shell)
+        dry.dryRun = true
+        def d:ShellResult (dry.run("rm" (ShellTest.two("-rf" "/"))))
+        this.check("a dry run does not run" (d.skipped))
+        this.check("and answers success" (d.ok()))
+        this.check("and records what it would have run" ((array_length dry.commands) == 1))
+        this.check("as a readable line" ((itemAt dry.commands 0) == "rm -rf /"))
+        def d2:ShellResult (dry.capture("ls" (ShellTest.one("-la"))))
+        this.check("captures are recorded too" ((array_length dry.commands) == 2))
+        this.check("and answer no output" ((d2.out == "") && (d2.err == "")))
+
+        ; Every command is logged whether it ran or not, which is what makes a
+        ; log a record rather than a highlight reel.
+        def logged:Shell (new Shell)
+        logged.capture("sh" (ShellTest.two("-c" "true")))
+        logged.capture("sh" (ShellTest.two("-c" "false")))
+        this.check("a real shell logs everything it ran" ((array_length logged.commands) == 2))
+        this.check("and remembers the last exit code" (logged.lastCode == 1))
+    }
+
+    sfn m@(main):void () {
+        def t:ShellTest (new ShellTest)
+        print "Shell and run_process_result"
+        t.run()
+        print "  " + t.passed + " passed, " + t.failed + " failed"
+        if (t.failed > 0) {
+            exit 1
+        }
+    }
+}