visaljs 0.0.0-stage → 0.0.2

package/runtime/src/gc.c

@@ -0,0 +1,803 @@
+/* Heap and garbage collector (ADR 0002).
+ *
+ * Allocation bits say which cells hold a value. They are exact whenever the
+ * world is stopped:
+ *   - sweeping a block sets them to its mark bits (only survivors);
+ *   - when a block becomes the allocation block of its class, every cell's
+ *     bit is set, because the allocator may hand out any of them;
+ *   - stopping the world clears the bits of the cells still on a free list.
+ * Only the allocation block of a class has cells on a free list.
+ */
+#include "js_gc.h"
+
+#include <setjmp.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "internal.h"
+#include "js_builtins.h"
+#include "jscontext.h"
+#include "os.h"
+
+const uint16_t js_size_class_bytes[JS_NUM_SIZE_CLASSES] = {
+    16, 32, 48, 64, 80, 96, 112, 128,
+    160, 192, 224, 256,
+    320, 384, 448, 512,
+    640, 768, 896, 1024,
+    1280, 1536, 1792, 2048,
+    2560, 3072, 3584, 4096,
+    5120, 6144, 7168, 8192,
+};
+
+struct JSTracer {
+    JSHeap *heap;
+    size_t  sp;               /* mark stack depth */
+    const void *stack_low;    /* the conservative scan covers [stack_low, stack_base) */
+};
+
+_Noreturn void js_fatal(const char *fmt, ...) {
+    char msg[512];
+    va_list ap;
+    va_start(ap, fmt);
+    int n = snprintf(msg, sizeof msg, "jsrt: fatal: ");
+    vsnprintf(msg + n, sizeof msg - (size_t)n - 1, fmt, ap);
+    va_end(ap);
+    strcat(msg, "\n");
+    fputs(msg, stderr);  /* one write, so the message arrives whole */
+    exit(70);            /* EX_SOFTWARE */
+}
+
+static void *xrealloc(void *p, size_t size) {
+    void *q = realloc(p, size);
+    if (!q) js_fatal("out of memory (%zu bytes of GC metadata)", size);
+    return q;
+}
+
+/* ADR 0001: every cell address must fit in 48 bits. */
+static void check_address(const void *p, size_t size) {
+    if ((uintptr_t)p + size > (UINT64_C(1) << 48))
+        js_fatal("heap memory mapped above 2^48 (%p)", p);
+}
+
+/* ---- Cell classes ------------------------------------------------------ */
+
+static void value_array_visit(JSCell *c, JSTracer *trc) {
+    JSValueArray *a = (JSValueArray *)c;
+    for (uint32_t i = 0; i < c->aux; i++) js_trace_value(trc, a->values[i]);
+}
+
+static const JSCellClass value_array_class = { "ValueArray", value_array_visit, NULL };
+
+static const JSCellClass bigint_class = { "BigInt", NULL, NULL };  /* limbs only (ADR 0016) */
+
+static const JSCellClass *const cell_classes[JS_CELL_KIND_COUNT] = {
+    [JS_CELL_STR_FLAT] = &js_flat_string_class,
+    [JS_CELL_STR_ROPE] = &js_rope_class,
+    [JS_CELL_STR_SLICE] = &js_slice_class,
+    [JS_CELL_VALUE_ARRAY] = &value_array_class,
+    [JS_CELL_SHAPE] = &js_shape_cell_class,
+    [JS_CELL_ACCESSOR] = &js_accessor_cell_class,
+    [JS_CELL_ELEMENTS] = &js_elements_cell_class,
+    [JS_CELL_ENV] = &js_env_cell_class,
+    [JS_CELL_SYMBOL] = &js_symbol_cell_class,
+    [JS_CELL_BIGINT] = &bigint_class,
+    [JS_CELL_OBJECT] = &js_object_cell_class,
+    [JS_CELL_ARRAY] = &js_object_cell_class,
+    [JS_CELL_FUNCTION] = &js_object_cell_class,
+};
+
+static const JSCellClass *class_of(JSCell *c) {
+    return c->kind == JS_CELL_HOST ? ((JSHostCell *)c)->cls : cell_classes[c->kind];
+}
+
+/* ---- Bitmaps ----------------------------------------------------------- */
+
+static char *block_cell(JSBlock *b, uint32_t i) {
+    return (char *)b + JS_BLOCK_FIRST_CELL + (size_t)i * b->cell_size;
+}
+
+static uint32_t cell_index(JSBlock *b, const void *c) {
+    return (uint32_t)(((const char *)c - (const char *)b - JS_BLOCK_FIRST_CELL) / b->cell_size);
+}
+
+static int bit_get(const uint64_t *bits, uint32_t i) { return (int)((bits[i / 64] >> (i % 64)) & 1); }
+static void bit_set(uint64_t *bits, uint32_t i) { bits[i / 64] |= UINT64_C(1) << (i % 64); }
+static void bit_clear(uint64_t *bits, uint32_t i) { bits[i / 64] &= ~(UINT64_C(1) << (i % 64)); }
+
+static void set_all_allocated(JSBlock *b) {
+    uint32_t full = b->cell_count / 64, rest = b->cell_count % 64;
+    memset(b->alloc_bits, 0xFF, full * sizeof(uint64_t));
+    if (rest) b->alloc_bits[full] = (UINT64_C(1) << rest) - 1;
+}
+
+/* ---- Block set: which 64 KiB chunks are small-object blocks ------------ */
+
+static size_t block_hash(uintptr_t b) {
+    uint64_t x = (uint64_t)(b >> 16) * UINT64_C(0x9E3779B97F4A7C15);
+    return (size_t)(x >> 32);
+}
+
+static void bloom_add(JSHeap *h, uintptr_t b) {
+    size_t bit = block_hash(b) & 255;
+    h->block_bloom[bit / 64] |= UINT64_C(1) << (bit % 64);
+}
+
+static int bloom_maybe(const JSHeap *h, uintptr_t b) {
+    size_t bit = block_hash(b) & 255;
+    return (int)((h->block_bloom[bit / 64] >> (bit % 64)) & 1);
+}
+
+static void block_set_put(uintptr_t *keys, size_t cap, uintptr_t b) {
+    size_t i = block_hash(b) & (cap - 1);
+    while (keys[i]) i = (i + 1) & (cap - 1);
+    keys[i] = b;
+}
+
+static void block_set_add(JSHeap *h, uintptr_t b) {
+    if (2 * (h->block_set_count + 1) > h->block_set_cap) {  /* load factor <= 1/2 */
+        size_t cap = h->block_set_cap ? h->block_set_cap * 2 : 64;
+        uintptr_t *keys = calloc(cap, sizeof *keys);
+        if (!keys) js_fatal("out of memory growing the block set");
+        for (size_t i = 0; i < h->block_set_cap; i++)
+            if (h->block_set[i]) block_set_put(keys, cap, h->block_set[i]);
+        free(h->block_set);
+        h->block_set = keys;
+        h->block_set_cap = cap;
+    }
+    block_set_put(h->block_set, h->block_set_cap, b);
+    h->block_set_count++;
+    bloom_add(h, b);
+}
+
+static int block_set_contains(const JSHeap *h, uintptr_t b) {
+    if (!h->block_set_cap || !bloom_maybe(h, b)) return 0;
+    for (size_t i = block_hash(b) & (h->block_set_cap - 1);; i = (i + 1) & (h->block_set_cap - 1)) {
+        if (h->block_set[i] == b) return 1;
+        if (!h->block_set[i]) return 0;
+    }
+}
+
+/* Linear probing: shift later entries of the probe run back into the hole. */
+static void block_set_remove(JSHeap *h, uintptr_t b) {
+    size_t mask = h->block_set_cap - 1, i = block_hash(b) & mask;
+    while (h->block_set[i] != b) {
+        if (!h->block_set[i]) return;
+        i = (i + 1) & mask;
+    }
+    h->block_set[i] = 0;
+    h->block_set_count--;
+    for (size_t j = (i + 1) & mask; h->block_set[j]; j = (j + 1) & mask) {
+        size_t home = block_hash(h->block_set[j]) & mask;
+        int stays = i <= j ? (i < home && home <= j) : (i < home || home <= j);
+        if (!stays) {
+            h->block_set[i] = h->block_set[j];
+            h->block_set[j] = 0;
+            i = j;
+        }
+    }
+    /* The Bloom filter keeps the bit: it only ever answers "maybe". */
+}
+
+static void note_mapping(JSHeap *h, const void *p, size_t size) {
+    uintptr_t lo = (uintptr_t)p, hi = lo + size;
+    if (!h->min_addr || lo < h->min_addr) h->min_addr = lo;
+    if (hi > h->max_addr) h->max_addr = hi;
+}
+
+/* ---- Small-object blocks ----------------------------------------------- */
+
+static JSBlock *map_block(JSHeap *h) {
+    JSBlock *b = js_os_map_aligned(JS_BLOCK_SIZE, JS_BLOCK_SIZE);
+    if (!b) js_fatal("out of memory mapping a heap block");
+    check_address(b, JS_BLOCK_SIZE);
+    note_mapping(h, b, JS_BLOCK_SIZE);
+    block_set_add(h, (uintptr_t)b);
+    h->block_count++;
+    return b;
+}
+
+/* Makes `b` the allocation block of its class with `list` (`n` cells) as
+ * the free list. */
+static void *start_allocating(JSHeap *h, JSBlock *b, JSFreeCell *list, uint32_t n) {
+    h->allocated_since_gc += (size_t)n * b->cell_size;
+    set_all_allocated(b);
+    h->current[b->size_class] = b;
+    h->free[b->size_class] = list->next;
+    return list;
+}
+
+/* Gives an empty block to size class `sc`; returns its first cell. */
+static void *block_init(JSHeap *h, JSBlock *b, unsigned sc) {
+    uint32_t size = js_size_class_bytes[sc];
+    memset(b, 0, sizeof *b);
+    b->map_size = JS_BLOCK_SIZE;
+    b->cell_size = size;
+    b->cell_count = (uint32_t)((JS_BLOCK_SIZE - JS_BLOCK_FIRST_CELL) / size);
+    b->size_class = (uint8_t)sc;
+
+    JSFreeCell *head = NULL;
+    for (uint32_t i = b->cell_count; i-- > 0;) {
+        JSFreeCell *c = (JSFreeCell *)block_cell(b, i);
+        c->next = head;
+        head = c;
+    }
+    b->next = h->blocks[sc];
+    h->blocks[sc] = b;
+    return start_allocating(h, b, head, b->cell_count);
+}
+
+static void finalize_cell(JSRuntime *rt, JSCell *c) {
+    const JSCellClass *cls = class_of(c);
+    if (cls->finalize) cls->finalize(rt, c);
+}
+
+/* Returns the dead cells of `b` as a free list (finalize_dead already ran their finalizers).
+ * Afterwards only the survivors count as allocated. */
+static JSFreeCell *sweep_block(JSBlock *b, uint32_t *live_out) {
+    JSFreeCell *head = NULL;
+    uint32_t live = 0;
+    for (uint32_t i = b->cell_count; i-- > 0;) {
+        if (bit_get(b->mark_bits, i)) {
+            live++;
+            continue;
+        }
+        JSFreeCell *f = (JSFreeCell *)block_cell(b, i);
+        f->next = head;
+        head = f;
+    }
+    memcpy(b->alloc_bits, b->mark_bits, sizeof b->alloc_bits);
+    b->unswept = 0;
+    if (!live) b->has_finalizable = 0;
+    *live_out = live;
+    return head;
+}
+
+static void unmap_block(JSHeap *h, JSBlock *b) {
+    block_set_remove(h, (uintptr_t)b);
+    js_os_unmap(b, JS_BLOCK_SIZE);
+    h->block_count--;
+}
+
+/* Keeps a few empty blocks for reuse by any class; the rest go back to the OS. */
+static void release_empty_block(JSHeap *h, JSBlock *b) {
+    if (h->empty_count >= JS_EMPTY_POOL_MAX) {
+        unmap_block(h, b);
+        return;
+    }
+    b->next = h->empty;
+    h->empty = b;
+    h->empty_count++;
+}
+
+/* Sweeps the next unswept block of class `sc`. Returns 0 when there is none;
+ * otherwise stores its free list (NULL if it had no dead cells) in *list and
+ * the list's length in *n. */
+static int sweep_next(JSRuntime *rt, unsigned sc, JSFreeCell **list, uint32_t *n) {
+    JSHeap *h = &rt->heap;
+    JSBlock **link = h->sweep_link[sc];
+    while (*link && !(*link)->unswept) link = &(*link)->next;
+    JSBlock *b = *link;
+    if (!b) {
+        h->sweep_link[sc] = link;
+        return 0;
+    }
+    uint32_t live;
+    JSFreeCell *free_list = sweep_block(b, &live);
+    *n = b->cell_count - live;  /* before b may be unmapped below */
+    if (!live) {
+        *link = b->next;  /* unlink; an empty block can serve any class */
+        release_empty_block(h, b);
+        free_list = NULL;
+    } else {
+        link = &b->next;
+    }
+    h->sweep_link[sc] = link;
+    *list = free_list;
+    return 1;
+}
+
+static int should_collect(const JSHeap *h) {
+    size_t threshold = h->live_after_gc > h->min_heap ? h->live_after_gc : h->min_heap;
+    return h->allocated_since_gc > threshold;
+}
+
+static int over_limit(const JSRuntime *rt, size_t more) {
+    return rt->heap.heap_limit && js_heap_size(rt) + more > rt->heap.heap_limit;
+}
+
+static _Noreturn void heap_limit_reached(const JSHeap *h) {
+    js_fatal("heap limit reached (%zu MiB)", h->heap_limit >> 20);
+}
+
+#ifdef JS_GC_STRESS
+/* Collects on every allocation (every JS_GC_STRESS_INTERVAL-th). */
+static void stress_tick(JSContext *ctx) {
+    JSHeap *h = &ctx->rt->heap;
+    if (!h->in_gc && ++h->stress_counter >= h->stress_interval) {
+        h->stress_counter = 0;
+        js_gc(ctx);
+    }
+}
+#endif
+
+/* Lazily sweeps blocks of class `sc` until one has a dead cell to reuse. */
+static void *sweep_for(JSRuntime *rt, unsigned sc) {
+    JSFreeCell *list;
+    uint32_t n;
+    while (sweep_next(rt, sc, &list, &n)) {
+        if (list) return start_allocating(&rt->heap, js_block_of(list), list, n);
+    }
+    return NULL;
+}
+
+void *js_alloc_slow(JSContext *ctx, unsigned sc) {
+    JSRuntime *rt = ctx->rt;
+    JSHeap *h = &rt->heap;
+#ifdef JS_GC_STRESS
+    stress_tick(ctx);
+    JSFreeCell *c = h->free[sc];
+    if (c) {
+        h->free[sc] = c->next;
+        return c;
+    }
+#endif
+    for (int collected = 0;; collected = 1) {
+        void *cell = sweep_for(rt, sc);
+        if (cell) return cell;
+        JSBlock *b = h->empty;
+        if (b) {
+            h->empty = b->next;
+            h->empty_count--;
+            return block_init(h, b, sc);
+        }
+        if (!collected && !h->in_gc && (should_collect(h) || over_limit(rt, JS_BLOCK_SIZE))) {
+            js_gc(ctx);
+            continue;
+        }
+        if (over_limit(rt, JS_BLOCK_SIZE)) heap_limit_reached(h);
+        return block_init(h, map_block(h), sc);
+    }
+}
+
+/* ---- Large objects ----------------------------------------------------- */
+
+/* Returns empty blocks to the OS to make room under the heap limit. */
+static void drain_empty_pool(JSHeap *h) {
+    while (h->empty) {
+        JSBlock *b = h->empty;
+        h->empty = b->next;
+        h->empty_count--;
+        unmap_block(h, b);
+    }
+}
+
+static JSCell *alloc_large(JSContext *ctx, size_t size) {
+    JSRuntime *rt = ctx->rt;
+    JSHeap *h = &rt->heap;
+    size_t page = js_os_page_size();
+    size_t map_size = (JS_BLOCK_FIRST_CELL + size + page - 1) & ~(page - 1);
+#ifdef JS_GC_STRESS
+    stress_tick(ctx);
+#endif
+    if (!h->in_gc && (should_collect(h) || over_limit(rt, map_size))) js_gc(ctx);  /* large objects are swept eagerly */
+    if (over_limit(rt, map_size)) drain_empty_pool(h);
+    if (over_limit(rt, map_size)) heap_limit_reached(h);
+    h->allocated_since_gc += map_size;
+    JSBlock *b = js_os_map_aligned(map_size, JS_BLOCK_SIZE);  /* zero-filled */
+    if (!b) js_fatal("out of memory allocating %zu bytes", size);
+    check_address(b, map_size);
+    note_mapping(h, b, map_size);
+    b->map_size = map_size;
+    b->cell_size = (uint32_t)(size <= UINT32_MAX ? size : UINT32_MAX);
+    b->cell_count = 1;
+    b->is_large = 1;
+    b->alloc_bits[0] = 1;
+
+    if (h->large_count == h->large_cap) {
+        h->large_cap = h->large_cap ? h->large_cap * 2 : 16;
+        h->large = xrealloc(h->large, h->large_cap * sizeof *h->large);
+    }
+    size_t i = h->large_count;  /* keep sorted by address */
+    while (i > 0 && h->large[i - 1] > b) {
+        h->large[i] = h->large[i - 1];
+        i--;
+    }
+    h->large[i] = b;
+    h->large_count++;
+    h->large_bytes += map_size;
+    return (JSCell *)block_cell(b, 0);
+}
+
+static void sweep_large(JSRuntime *rt) {
+    JSHeap *h = &rt->heap;
+    size_t kept = 0;
+    for (size_t i = 0; i < h->large_count; i++) {
+        JSBlock *b = h->large[i];
+        if (b->mark_bits[0] & 1) {
+            h->large[kept++] = b;
+            continue;
+        }
+        h->large_bytes -= b->map_size;
+        js_os_unmap(b, b->map_size);
+    }
+    h->large_count = kept;
+}
+
+/* ---- Allocation entry points ------------------------------------------- */
+
+JSCell *js_alloc_cell(JSContext *ctx, JSCellKind kind, size_t size) {
+    JSCell *c = size <= JS_MAX_SMALL_SIZE ? js_alloc_small(ctx, js_size_class(size))
+                                          : alloc_large(ctx, size);
+    c->kind = (uint8_t)kind;
+    return c;
+}
+
+JSValueArray *js_alloc_value_array(JSContext *ctx, uint32_t length) {
+    JSCell *c = js_alloc_cell(ctx, JS_CELL_VALUE_ARRAY,
+                              sizeof(JSValueArray) + (size_t)length * sizeof(JSValue));
+    c->aux = length;
+    return (JSValueArray *)c;
+}
+
+JSHostCell *js_alloc_host_cell(JSContext *ctx, const JSCellClass *cls, size_t size) {
+    if (size < sizeof(JSHostCell)) size = sizeof(JSHostCell);
+    JSHostCell *h = (JSHostCell *)js_alloc_cell(ctx, JS_CELL_HOST, size);
+    h->cls = cls;
+    if (cls->finalize) {
+        h->cell.flags |= JS_CELL_FINALIZABLE;
+        js_block_of(h)->has_finalizable = 1;
+    }
+    return h;
+}
+
+/* ---- Roots ------------------------------------------------------------- */
+
+void js_register_roots(JSRuntime *rt, JSValue *slots, size_t n) {
+    JSHeap *h = &rt->heap;
+    if (h->root_table_count == h->root_table_cap) {
+        h->root_table_cap = h->root_table_cap ? h->root_table_cap * 2 : 8;
+        h->root_tables = xrealloc(h->root_tables, h->root_table_cap * sizeof *h->root_tables);
+    }
+    h->root_tables[h->root_table_count++] = (struct JSRootTable){ slots, n };
+}
+
+void js_add_root(JSRuntime *rt, JSValue *slot) {
+    JSHeap *h = &rt->heap;
+    if (h->root_count == h->root_cap) {
+        h->root_cap = h->root_cap ? h->root_cap * 2 : 16;
+        h->roots = xrealloc(h->roots, h->root_cap * sizeof *h->roots);
+    }
+    h->roots[h->root_count++] = slot;
+}
+
+void js_remove_root(JSRuntime *rt, JSValue *slot) {
+    JSHeap *h = &rt->heap;
+    for (size_t i = h->root_count; i-- > 0;) {
+        if (h->roots[i] == slot) {
+            h->roots[i] = h->roots[--h->root_count];
+            return;
+        }
+    }
+}
+
+/* ---- Marking ----------------------------------------------------------- */
+
+void js_trace_cell(JSTracer *trc, JSCell *c) {
+    if (js_cell_is_static(c)) return;
+    JSBlock *b = js_block_of(c);
+    uint32_t i = cell_index(b, c);
+    if (bit_get(b->mark_bits, i)) return;
+    bit_set(b->mark_bits, i);
+    JSHeap *h = trc->heap;
+    if (trc->sp == h->mark_stack_cap) {
+        h->mark_stack_cap = h->mark_stack_cap ? h->mark_stack_cap * 2 : 4096;
+        h->mark_stack = xrealloc(h->mark_stack, h->mark_stack_cap * sizeof *h->mark_stack);
+    }
+    h->mark_stack[trc->sp++] = c;
+}
+
+bool js_gc_is_marked(const JSCell *c) {
+    if (js_cell_is_static(c)) return true;
+    JSBlock *b = js_block_of(c);
+    return bit_get(b->mark_bits, cell_index(b, c));
+}
+
+void js_trace_value(JSTracer *trc, JSValue v) {
+    if (js_is_cell(v)) js_trace_cell(trc, js_as_cell(v));
+}
+
+/* Explicit stack, never recursion: ropes and linked lists can be very deep. */
+static void drain(JSTracer *trc) {
+    while (trc->sp) {
+        JSCell *c = trc->heap->mark_stack[--trc->sp];
+        const JSCellClass *cls = class_of(c);
+        if (cls->visit) cls->visit(c, trc);
+    }
+}
+
+static void mark_roots(JSRuntime *rt, JSTracer *trc) {
+    JSHeap *h = &rt->heap;
+    js_trace_value(trc, rt->exception);  /* ADR 0004, invariant 4 */
+    js_trace_value(trc, rt->symbol_registry);
+    js_jobs_trace(rt, trc);  /* queued jobs, unhandled rejections (ADR 0008) */
+    js_weak_trace_roots(rt, trc);  /* the WeakRef kept-alive set */
+    for (JSContext *realm = rt->realms; realm; realm = realm->next_realm)
+        for (unsigned i = 0; i < JS_INTRINSIC_COUNT; i++) js_trace_value(trc, realm->intrinsics[i]);
+    for (size_t t = 0; t < h->root_table_count; t++)
+        for (size_t i = 0; i < h->root_tables[t].n; i++) js_trace_value(trc, h->root_tables[t].slots[i]);
+    for (size_t i = 0; i < h->root_count; i++) js_trace_value(trc, *h->roots[i]);
+}
+
+/* ---- Conservative roots ------------------------------------------------ */
+
+static JSBlock *find_large(const JSHeap *h, uintptr_t w) {
+    size_t lo = 0, hi = h->large_count;
+    while (lo < hi) {  /* last object starting at or below w */
+        size_t mid = (lo + hi) / 2;
+        if ((uintptr_t)h->large[mid] <= w) lo = mid + 1;
+        else hi = mid;
+    }
+    if (!lo) return NULL;
+    JSBlock *b = h->large[lo - 1];
+    return w < (uintptr_t)b + b->map_size ? b : NULL;
+}
+
+/* The allocated cell that `w` points into (interior pointers included), or NULL. */
+static JSCell *find_cell(const JSHeap *h, uintptr_t w) {
+    if (w < h->min_addr || w >= h->max_addr) return NULL;  /* also rejects doubles, int32s */
+    uintptr_t base = w & ~(uintptr_t)(JS_BLOCK_SIZE - 1);
+    JSBlock *b;
+    uint32_t i;
+    if (block_set_contains(h, base)) {
+        b = (JSBlock *)base;
+        if (w < base + JS_BLOCK_FIRST_CELL) return NULL;
+        i = (uint32_t)((w - base - JS_BLOCK_FIRST_CELL) / b->cell_size);
+        if (i >= b->cell_count) return NULL;
+    } else {
+        b = find_large(h, w);
+        if (!b || w < (uintptr_t)b + JS_BLOCK_FIRST_CELL) return NULL;
+        i = 0;
+    }
+    return bit_get(b->alloc_bits, i) ? (JSCell *)block_cell(b, i) : NULL;
+}
+
+#if defined(__clang__) || defined(__GNUC__)
+#define JS_NOINLINE __attribute__((noinline))
+#define JS_NO_ASAN __attribute__((no_sanitize_address))
+#else
+#define JS_NOINLINE __declspec(noinline)
+#define JS_NO_ASAN
+#endif
+
+#if defined(__SANITIZE_ADDRESS__)
+#define JS_ASAN 1
+#elif defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define JS_ASAN 1
+#endif
+#endif
+
+#ifdef JS_ASAN
+/* ASan's use-after-return mode moves locals to a "fake stack" in heap memory,
+ * where the conservative scan can't see them. Keep them on the real stack. */
+const char *__asan_default_options(void);
+const char *__asan_default_options(void) { return "detect_stack_use_after_return=0"; }
+#endif
+
+/* Scans the mutator's stack: from js_gc's frame (which holds the spilled
+ * registers) up to the stack base. The collector's own deeper frames are
+ * not roots, and would hold stale pointers from sweeping.
+ * Reads the whole range on purpose, redzones included. */
+static JS_NO_ASAN void mark_stack(JSTracer *trc) {
+    const uintptr_t *p = (const uintptr_t *)((uintptr_t)trc->stack_low & ~(uintptr_t)(sizeof(uintptr_t) - 1));
+    const uintptr_t *end = (const uintptr_t *)trc->heap->stack_base;
+    for (; p < end; p++) {
+        JSCell *c = find_cell(trc->heap, *p);
+        if (c) js_trace_cell(trc, c);
+    }
+}
+
+/* ---- Collection -------------------------------------------------------- */
+
+/* Makes the allocation bits exact: cells still on a free list aren't allocated. */
+static void stop_allocation(JSHeap *h) {
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++) {
+        JSBlock *b = h->current[sc];
+        for (JSFreeCell *c = h->free[sc]; c; c = c->next) bit_clear(b->alloc_bits, cell_index(b, c));
+        h->free[sc] = NULL;
+        h->current[sc] = NULL;
+    }
+}
+
+/* Unswept blocks may hold dead cells whose children are already reused.
+ * Sweep them before marking so that nothing can resurrect such a cell. */
+static void finish_sweeping(JSRuntime *rt) {
+    JSFreeCell *list;
+    uint32_t n;
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++)
+        while (sweep_next(rt, sc, &list, &n)) {}
+}
+
+static unsigned popcount64(uint64_t x) {
+#if defined(_MSC_VER) && !defined(__clang__)
+    return (unsigned)__popcnt64(x);
+#else
+    return (unsigned)__builtin_popcountll(x);
+#endif
+}
+
+static size_t marked_bytes(const JSHeap *h) {
+    size_t bytes = 0;
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++)
+        for (JSBlock *b = h->blocks[sc]; b; b = b->next) {
+            size_t cells = 0;
+            for (unsigned w = 0; w < JS_BITMAP_WORDS; w++) cells += popcount64(b->mark_bits[w]);
+            bytes += cells * b->cell_size;
+        }
+    return bytes + h->large_bytes;  /* after sweep_large: only survivors remain */
+}
+
+/* Runs the finalizers of the dead cells, all of them, before any memory is freed or reused. A
+ * finalizer may read other cells: an object's reaches its class through its shape, which may be
+ * dead too, and which lazy sweeping could otherwise free and reuse before the object's own block
+ * is swept. Each finalized cell loses JS_CELL_FINALIZABLE, so nothing finalizes it twice. */
+static void finalize_dead(JSRuntime *rt) {
+    JSHeap *h = &rt->heap;
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++)
+        for (JSBlock *b = h->blocks[sc]; b; b = b->next) {
+            if (!b->has_finalizable) continue;
+            for (uint32_t i = 0; i < b->cell_count; i++) {
+                JSCell *c = (JSCell *)block_cell(b, i);
+                if (bit_get(b->alloc_bits, i) && !bit_get(b->mark_bits, i) && (c->flags & JS_CELL_FINALIZABLE)) {
+                    finalize_cell(rt, c);
+                    c->flags &= (uint16_t)~JS_CELL_FINALIZABLE;
+                }
+            }
+        }
+    for (size_t i = 0; i < h->large_count; i++) {
+        JSBlock *b = h->large[i];
+        JSCell *c = (JSCell *)block_cell(b, 0);
+        if (!(b->mark_bits[0] & 1) && (c->flags & JS_CELL_FINALIZABLE)) {
+            finalize_cell(rt, c);
+            c->flags &= (uint16_t)~JS_CELL_FINALIZABLE;
+        }
+    }
+}
+
+static JS_NOINLINE void collect(JSContext *ctx, const void *stack_low) {
+    JSRuntime *rt = ctx->rt;
+    JSHeap *h = &rt->heap;
+    if (h->in_gc) js_fatal("js_gc called during a collection (from a finalizer?)");
+    h->in_gc = 1;
+    stop_allocation(h);
+    finish_sweeping(rt);
+
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++)
+        for (JSBlock *b = h->blocks[sc]; b; b = b->next) memset(b->mark_bits, 0, sizeof b->mark_bits);
+    for (size_t i = 0; i < h->large_count; i++) h->large[i]->mark_bits[0] = 0;
+
+    JSTracer trc = { h, 0, stack_low };
+    mark_roots(rt, &trc);
+    mark_stack(&trc);
+    drain(&trc);
+    while (js_weak_ephemerons(rt, &trc)) drain(&trc);  /* a WeakMap value lives if its key does */
+    js_weak_sweep(rt);
+    js_atoms_sweep(rt);  /* weak references, before anything is freed */
+    js_shapes_sweep(rt);
+    js_prop_caches_sweep(rt);
+
+    finalize_dead(rt);
+    sweep_large(rt);
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++) {
+        for (JSBlock *b = h->blocks[sc]; b; b = b->next) b->unswept = 1;
+        h->sweep_link[sc] = &h->blocks[sc];
+    }
+    h->live_after_gc = marked_bytes(h);
+    h->allocated_since_gc = 0;
+    h->gc_count++;
+    h->in_gc = 0;
+}
+
+/* Spills the mutator's callee-saved registers into this frame, then
+ * collects in deeper frames that the stack scan doesn't cover. */
+JS_NOINLINE void js_gc(JSContext *ctx) {
+    jmp_buf regs;
+    memset(&regs, 0, sizeof regs);  /* no stale words in the parts setjmp skips */
+#if defined(__clang__) || defined(__GNUC__)
+    __builtin_unwind_init();
+#endif
+    setjmp(regs);
+    collect(ctx, &regs);
+    (void)*(volatile char *)&regs;  /* keep this frame alive: no tail call */
+}
+
+/* ---- Runtime lifetime -------------------------------------------------- */
+
+JSRuntime *js_runtime_new(void) {
+    JSRuntime *rt = calloc(1, sizeof *rt);
+    if (!rt) js_fatal("out of memory creating a runtime");
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++) rt->heap.sweep_link[sc] = &rt->heap.blocks[sc];
+    void *stack_low;
+    js_os_stack_bounds(&stack_low, &rt->heap.stack_base);
+    rt->stack_limit = (uintptr_t)stack_low + JS_STACK_MARGIN;
+    rt->exception = JS_EMPTY;
+    rt->symbol_registry = JS_UNDEFINED;
+    rt->heap.min_heap = JS_GC_MIN_HEAP;
+    const char *limit = getenv("JS_HEAP_LIMIT_MB");
+    if (limit) rt->heap.heap_limit = (size_t)strtoull(limit, NULL, 10) << 20;
+    const char *interval = getenv("JS_GC_STRESS_INTERVAL");
+    rt->heap.stress_interval = interval ? (size_t)strtoull(interval, NULL, 10) : 1;
+    js_atoms_init(rt);
+    return rt;
+}
+
+void js_set_heap_limit(JSRuntime *rt, size_t bytes) {
+    rt->heap.heap_limit = bytes;
+}
+
+size_t js_heap_size(const JSRuntime *rt) {
+    return rt->heap.block_count * JS_BLOCK_SIZE + rt->heap.large_bytes;
+}
+
+/* Runs the finalizers of every allocated cell in `b` that has one. */
+static void finalize_block(JSRuntime *rt, JSBlock *b) {
+    if (!b->has_finalizable) return;
+    for (uint32_t i = 0; i < b->cell_count; i++) {
+        JSCell *c = (JSCell *)block_cell(b, i);
+        if (bit_get(b->alloc_bits, i) && (c->flags & JS_CELL_FINALIZABLE)) finalize_cell(rt, c);
+    }
+}
+
+void js_runtime_free(JSRuntime *rt) {
+    JSHeap *h = &rt->heap;
+    stop_allocation(h);
+    /* every finalizer first, then the memory: a finalizer may read cells in other blocks (shapes) */
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++)
+        for (JSBlock *b = h->blocks[sc]; b; b = b->next) finalize_block(rt, b);
+    for (size_t i = 0; i < h->large_count; i++) finalize_block(rt, h->large[i]);
+    for (unsigned sc = 0; sc < JS_NUM_SIZE_CLASSES; sc++) {
+        for (JSBlock *b = h->blocks[sc], *next; b; b = next) {
+            next = b->next;
+            js_os_unmap(b, JS_BLOCK_SIZE);
+        }
+    }
+    for (JSBlock *b = h->empty, *next; b; b = next) {
+        next = b->next;
+        js_os_unmap(b, JS_BLOCK_SIZE);
+    }
+    for (size_t i = 0; i < h->large_count; i++) js_os_unmap(h->large[i], h->large[i]->map_size);
+    js_atoms_free(rt);
+    js_shapes_free(rt);
+    js_prop_caches_free(rt);
+    js_jobs_free(rt);
+    js_weak_free(rt);
+    while (rt->realms) js_context_free(rt->realms);  /* e.g. $262.createRealm()'s */
+    free(h->large);
+    free(h->block_set);
+    free(h->root_tables);
+    free(h->roots);
+    free(h->mark_stack);
+    free(rt);
+}
+
+JSContext *js_context_new(JSRuntime *rt) {
+    JSContext *ctx = calloc(1, sizeof *ctx);
+    if (!ctx) js_fatal("out of memory creating a context");
+    ctx->rt = rt;
+    ctx->stack_limit = rt->stack_limit;
+    for (unsigned i = 0; i < JS_INTRINSIC_COUNT; i++) ctx->intrinsics[i] = JS_UNDEFINED;
+    ctx->next_realm = rt->realms;
+    rt->realms = ctx;
+    js_realm_init(ctx);
+    js_install_recorded_components(ctx);  /* the program's optional built-ins (ADR 0014) */
+    return ctx;
+}
+
+void js_context_free(JSContext *ctx) {
+    JSContext **link = &ctx->rt->realms;
+    while (*link != ctx) link = &(*link)->next_realm;
+    *link = ctx->next_realm;
+    free(ctx);
+}