xbintsc 0.5.20 → 0.5.21

package/runtime/xt_fetch/request.inc

@@ -1,9 +1,10 @@
 /* -- request / response exchange ------------------------------------------ */
 
 /*
- * The actual `fetch` implementation. Because the runtime has no event loop,
- * this is a synchronous, blocking HTTP/1.1 exchange; the promise it returns is
- * already settled, which is all the current `await` machinery needs.
+ * Request serialisation, response framing and header parsing for `fetch`. The
+ * blocking exchange itself runs on an async I/O worker (see `async.inc`); these
+ * helpers are shared by the main thread (building the request, parsing the
+ * completed response) and the worker (deciding when the response is complete).
  */
 
 /** Index just past the blank line that terminates the response headers. */
@@ -204,183 +205,5 @@ static void xt_fetch_build_request(const char *method, const xt_fetch_url *url,
   if (body && body->length > 0) xt_fetch_buffer_append(out, body->data, body->length);
 }
 
-static xt_value xt_fetch_reject(const char *message) {
-  xt_value argument = xt_string_from_cstr(message);
-  return xt_promise_reject(xt_type_error_ctor(1, &argument));
-}
-
-xt_value xt_fetch(int32_t argc, xt_value *argv) {
-  xt_value input = xt_arg_at(argc, argv, 0);
-  xt_value init = xt_arg_at(argc, argv, 1);
-  if (input == XT_UNDEFINED || input == XT_NULL) return xt_fetch_reject("Failed to parse URL from undefined");
-
-  xt_value urlValue = input;
-  if (XT_IS_OBJECT(input)) {
-    xt_value candidate = xt_object_get_cstr(input, "url");
-    if (XT_IS_STRING(candidate)) urlValue = candidate;
-  }
-  const char *urlText = xt_string_data(xt_to_string(urlValue));
-  if (!urlText) return xt_fetch_reject("Failed to parse URL");
-
-  char currentUrl[4096];
-  snprintf(currentUrl, sizeof(currentUrl), "%s", urlText);
-
-  char method[16];
-  snprintf(method, sizeof(method), "GET");
-  xt_value headersSource = XT_UNDEFINED;
-  xt_value bodySource = XT_UNDEFINED;
-  const char *redirectMode = "follow";
-  if (XT_IS_OBJECT(init)) {
-    xt_value methodValue = xt_object_get_cstr(init, "method");
-    if (XT_IS_STRING(methodValue)) {
-      const char *text = xt_string_data(methodValue);
-      size_t i = 0;
-      for (; text && text[i] && i + 1 < sizeof(method); i++) {
-        char c = text[i];
-        method[i] = (c >= 'a' && c <= 'z') ? (char)(c - 32) : c;
-      }
-      method[i] = '\0';
-    }
-    headersSource = xt_object_get_cstr(init, "headers");
-    bodySource = xt_object_get_cstr(init, "body");
-    xt_value redirectValue = xt_object_get_cstr(init, "redirect");
-    if (XT_IS_STRING(redirectValue)) redirectMode = xt_string_data(redirectValue);
-  }
-
-  xt_fetch_buffer body;
-  xt_fetch_buffer_init(&body);
-  if (bodySource != XT_UNDEFINED && bodySource != XT_NULL) {
-    xt_value text = xt_to_string(bodySource);
-    const char *data = xt_string_data(text);
-    if (data) xt_fetch_buffer_append(&body, data, (size_t)xt_string_length_value(text));
-  }
-
-  for (int redirects = 0; redirects <= 20; redirects++) {
-    xt_fetch_url parsed;
-    if (xt_fetch_parse_url(currentUrl, &parsed) != 0) {
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("Failed to parse URL");
-    }
-    if (parsed.secure) {
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("fetch failed: https is not supported");
-    }
-
-    int fd = xt_fetch_connect(&parsed);
-    if (fd < 0) {
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("fetch failed");
-    }
-
-    xt_value requestHeaders = xt_headers_from_source(headersSource);
-    xt_fetch_buffer request;
-    xt_fetch_buffer_init(&request);
-    xt_fetch_build_request(method, &parsed, requestHeaders, body.length > 0 ? &body : NULL, &request);
-    if (xt_fetch_send_all(fd, request.data, request.length) != 0) {
-      xt_fetch_close_socket(fd);
-      xt_fetch_buffer_free(&request);
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("fetch failed");
-    }
-    xt_fetch_buffer_free(&request);
-
-    xt_fetch_buffer raw;
-    xt_fetch_buffer_init(&raw);
-    char chunk[16384];
-    for (;;) {
-      int headEnd = xt_fetch_head_end(&raw);
-      if (headEnd > 0) {
-        int status = 0;
-        char statusText[256];
-        size_t headerStart = 0;
-        if (xt_fetch_parse_status_line(raw.data, raw.length, &status, statusText, sizeof(statusText), &headerStart) == 0) {
-          int stop = 0;
-          if (status == 204 || status == 304 || (status >= 100 && status < 200) || strcmp(method, "HEAD") == 0) {
-            stop = 1;
-          } else {
-            size_t contentLength = 0;
-            int hasContentLength = 0;
-            int chunked = 0;
-            xt_fetch_scan_headers(raw.data, headerStart, (size_t)headEnd - 2, &contentLength, &hasContentLength, &chunked);
-            if (chunked) {
-              size_t decoded = 0;
-              if (xt_fetch_chunked_complete(raw.data, raw.length, (size_t)headEnd, &decoded) == 1) stop = 1;
-            } else if (hasContentLength && raw.length - (size_t)headEnd >= contentLength) {
-              stop = 1;
-            }
-          }
-          if (stop) break;
-        }
-      }
-      int count = xt_fetch_recv(fd, chunk, sizeof(chunk));
-      if (count > 0) {
-        xt_fetch_buffer_append(&raw, chunk, (size_t)count);
-        continue;
-      }
-      break;
-    }
-    xt_fetch_close_socket(fd);
-
-    int headEnd = xt_fetch_head_end(&raw);
-    int status = 0;
-    char statusText[256];
-    size_t headerStart = 0;
-    if (headEnd <= 0 ||
-        xt_fetch_parse_status_line(raw.data, raw.length, &status, statusText, sizeof(statusText), &headerStart) != 0) {
-      xt_fetch_buffer_free(&raw);
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("fetch failed: invalid response");
-    }
-
-    xt_value responseHeaders = xt_headers_new();
-    xt_headers_parse_block(responseHeaders, raw.data + headerStart, (size_t)headEnd - 2 - headerStart);
-
-    int isRedirect = status == 301 || status == 302 || status == 303 || status == 307 || status == 308;
-    if (isRedirect && strcmp(redirectMode, "error") == 0) {
-      xt_fetch_buffer_free(&raw);
-      xt_fetch_buffer_free(&body);
-      return xt_fetch_reject("fetch failed: redirect not allowed");
-    }
-    if (isRedirect && strcmp(redirectMode, "manual") != 0 && redirects < 20) {
-      const char *location = xt_headers_lookup(responseHeaders, "location");
-      char resolved[4096];
-      if (location && xt_fetch_resolve_location(&parsed, location, resolved, sizeof(resolved)) == 0) {
-        if (status == 303 || ((status == 301 || status == 302) && strcmp(method, "POST") == 0)) {
-          snprintf(method, sizeof(method), "GET");
-          body.length = 0;
-        }
-        snprintf(currentUrl, sizeof(currentUrl), "%s", resolved);
-        xt_fetch_buffer_free(&raw);
-        continue;
-      }
-    }
-
-    size_t contentLength = 0;
-    int hasContentLength = 0;
-    int chunked = 0;
-    xt_fetch_scan_headers(raw.data, headerStart, (size_t)headEnd - 2, &contentLength, &hasContentLength, &chunked);
-    xt_fetch_buffer responseBody;
-    xt_fetch_buffer_init(&responseBody);
-    if (status == 204 || status == 304 || (status >= 100 && status < 200) || strcmp(method, "HEAD") == 0) {
-      /* No body by definition. */
-    } else if (chunked) {
-      xt_fetch_decode_chunked(raw.data, raw.length, (size_t)headEnd, &responseBody);
-    } else if (hasContentLength) {
-      size_t available = raw.length - (size_t)headEnd;
-      size_t take = contentLength < available ? contentLength : available;
-      xt_fetch_buffer_append(&responseBody, raw.data + headEnd, take);
-    } else {
-      xt_fetch_buffer_append(&responseBody, raw.data + headEnd, raw.length - (size_t)headEnd);
-    }
-
-    xt_value response = xt_response_new(responseHeaders, status, statusText, currentUrl,
-                                        responseBody.data ? responseBody.data : "", responseBody.length);
-    xt_fetch_buffer_free(&responseBody);
-    xt_fetch_buffer_free(&raw);
-    xt_fetch_buffer_free(&body);
-    return xt_promise_resolve(response);
-  }
-
-  xt_fetch_buffer_free(&body);
-  return xt_fetch_reject("fetch failed: too many redirects");
-}
+/* The async exchange itself lives in `async.inc`; it reuses the parsers and
+ * request builder above. */

package/runtime/xt_fetch/socket.inc

@@ -1,10 +1,11 @@
 /* -- TCP socket helpers --------------------------------------------------- */
 
 /*
- * `fetch` performs a conventional blocking request/response exchange: connect,
- * write the request, read until the response is complete (or the peer closes).
- * The socket carries a receive timeout so a stalled server cannot hang the
- * program forever; the timeout surfaces as a rejected promise.
+ * `fetch` performs a conventional blocking request/response exchange on an
+ * async I/O worker: connect, write the request, read until the response is
+ * complete (or the peer closes). The socket carries a receive timeout so a
+ * stalled server cannot pin a worker forever; the timeout surfaces as a
+ * rejected promise.
  */
 #define XT_FETCH_TIMEOUT_SECONDS 30
 

package/runtime/xt_fetch.c

@@ -2,10 +2,11 @@
  * xbintsc runtime — the global `fetch` API.
  *
  * This is a small, dependency-free HTTP/1.1 client exposed as the platform
- * `fetch` global. The runtime has no event loop (promises are drained
- * synchronously by `await`), so the request is performed with blocking sockets
- * and the returned promise is already settled. TLS is not implemented, so only
- * `http://` URLs are accepted; `https://` rejects with a `TypeError`.
+ * `fetch` global. The blocking socket exchange runs on the async I/O pool
+ * (`xt_async_io.c`), so a request in flight does not stall the reactor: timers,
+ * sockets and other `await`s keep running until the response is dispatched on
+ * the main thread. TLS is not implemented, so only `http://` URLs are accepted;
+ * `https://` rejects with a `TypeError`.
  *
  * The implementation is grouped by concern under `xt_fetch/` and `#include`d
  * here as a single translation unit, following the other runtime modules.
@@ -37,3 +38,4 @@ typedef socklen_t xt_fetch_socklen_t;
 #include "xt_fetch/headers.inc"
 #include "xt_fetch/response.inc"
 #include "xt_fetch/request.inc"
+#include "xt_fetch/async.inc"

package/runtime/xt_loop.c

@@ -33,6 +33,10 @@
 
 #define XT_LOOP_MAX_WATCHERS 1024
 
+/* Fallback wait bound while offloaded I/O is outstanding; the completion
+ * channel normally wakes `select` immediately. */
+#define XT_ASYNC_POLL_MS 10
+
 typedef struct {
   int fd;
   int events;
@@ -294,6 +298,11 @@ xt_value xt_clear_immediate(int32_t argc, xt_value *argv) {
   return xt_clear_timeout(argc, argv);
 }
 
+/* Reset to 1 while the reactor is dispatching handlers. Reusing a removed slot
+ * mid-dispatch would make the new watcher observe the current `fd_set`, so a
+ * registration from inside a callback appends instead. */
+static int xt_loop_dispatching = 0;
+
 int xt_loop_add(int fd, int events, xt_io_handler handler, void *userdata) {
   if (fd < 0 || !handler) return -1;
   xt_loop_gc_register();
@@ -304,6 +313,21 @@ int xt_loop_add(int fd, int events, xt_io_handler handler, void *userdata) {
     existing->userdata = userdata;
     return 0;
   }
+  /* Reuse a slot left by a removed watcher (the async pool toggles its wakeup
+   * descriptor on and off, and sockets are opened and closed repeatedly). */
+  if (!xt_loop_dispatching) {
+    for (int i = 0; i < xt_loop_watcher_count; i++) {
+      xt_loop_watcher *slot = &xt_loop_watchers[i];
+      if (slot->active) continue;
+      slot->fd = fd;
+      slot->events = events;
+      slot->handler = handler;
+      slot->userdata = userdata;
+      slot->active = 1;
+      xt_loop_active_count++;
+      return 0;
+    }
+  }
   if (xt_loop_watcher_count >= XT_LOOP_MAX_WATCHERS) return -1;
   xt_loop_watcher *watcher = &xt_loop_watchers[xt_loop_watcher_count++];
   watcher->fd = fd;
@@ -337,7 +361,8 @@ void xt_loop_set_main(xt_main_loop_fn fn) { xt_loop_main = fn; }
 int xt_loop_poll(int timeout_ms) {
   int has_fds = xt_loop_active_count > 0;
   int64_t next_due = xt_timer_next_due();
-  if (!has_fds && next_due < 0) {
+  int async_pending = xt_async_pending() > 0;
+  if (!has_fds && next_due < 0 && !async_pending) {
     xt_drain_microtasks();
     return 0;
   }
@@ -350,11 +375,15 @@ int xt_loop_poll(int timeout_ms) {
     if (delta > 100000000) delta = 100000000;
     if (wait_ms < 0 || delta < (int64_t)wait_ms) wait_ms = (int)delta;
   }
+  /* The completion channel normally wakes `select` immediately; polling every
+   * few milliseconds keeps progress even if it could not be registered. */
+  if (async_pending && (wait_ms < 0 || wait_ms > XT_ASYNC_POLL_MS)) wait_ms = XT_ASYNC_POLL_MS;
 
   /* Timers only: sleep, then run whatever is due. */
   if (!has_fds) {
     xt_loop_sleep_ms(wait_ms);
     xt_timer_run_due();
+    xt_async_poll();
     xt_drain_microtasks();
     return (int)(xt_loop_active_count + xt_timer_active_count());
   }
@@ -372,7 +401,9 @@ int xt_loop_poll(int timeout_ms) {
     if (watcher->fd > maxFd) maxFd = watcher->fd;
   }
   if (maxFd < 0) {
+    xt_loop_sleep_ms(wait_ms);
     xt_timer_run_due();
+    xt_async_poll();
     xt_drain_microtasks();
     return (int)(xt_loop_active_count + xt_timer_active_count());
   }
@@ -390,7 +421,13 @@ int xt_loop_poll(int timeout_ms) {
 
   if (ready > 0) {
     /* Collect the ready watchers first: handlers may add/remove descriptors. */
-    for (int i = 0; i < xt_loop_watcher_count; i++) {
+    int previousDispatching = xt_loop_dispatching;
+    xt_loop_dispatching = 1;
+    /* Snapshot the table: a handler may register a new descriptor, and it must
+     * wait for the next iteration rather than being tested against this
+     * iteration's `fd_set`. */
+    int dispatchCount = xt_loop_watcher_count;
+    for (int i = 0; i < dispatchCount; i++) {
       xt_loop_watcher *watcher = &xt_loop_watchers[i];
       if (!watcher->active) continue;
       int events = 0;
@@ -398,9 +435,11 @@ int xt_loop_poll(int timeout_ms) {
       if ((watcher->events & XT_IO_WRITE) && FD_ISSET(watcher->fd, &writeSet)) events |= XT_IO_WRITE;
       if (events) watcher->handler(watcher->userdata, events);
     }
+    xt_loop_dispatching = previousDispatching;
   }
 
   xt_timer_run_due();
+  xt_async_poll();
   xt_drain_microtasks();
   return (int)(xt_loop_active_count + xt_timer_active_count());
 }
@@ -417,4 +456,6 @@ void xt_run_event_loop(void) {
   }
 }
 
-int xt_loop_has_work(void) { return xt_loop_active_count > 0 || xt_timer_active_count() > 0; }
+int xt_loop_has_work(void) {
+  return xt_loop_active_count > 0 || xt_timer_active_count() > 0 || xt_async_pending() > 0;
+}

package/src/driver/compiler.ts

@@ -259,7 +259,7 @@ export function build(entryPath: string, options: BuildOptions = {}): BuildResul
     outputPath,
     linkerFlags: [
       ...toolchain.linkerArgs,
-      ...(process.platform === "win32" ? ["-lws2_32"] : ["-lm"]),
+      ...(process.platform === "win32" ? ["-lws2_32"] : ["-lm", "-pthread"]),
       ...registry.linkerFlags(),
       /* A GUI program is a Windows-subsystem executable: no console window is
        * created when it is launched. Console output (stdout/stderr) is lost in

package/src/driver/runtime-objects.ts

@@ -27,6 +27,7 @@ export const RUNTIME_SOURCES = [
   "xt_promise.c",
   "xt_async.c",
   "xt_loop.c",
+  "xt_async_io.c",
   "xt_generator.c",
   "xt_coroutine.c",
   "xt_fetch.c",

package/src/driver/version.ts

@@ -9,4 +9,4 @@
  * `main`/`master` only (see `.githooks/pre-commit`), so feature branches never
  * change this line and merging two of them cannot conflict here.
  */
-export const COMPILER_VERSION = "0.5.20";
+export const COMPILER_VERSION = "0.5.21";

package/src/extensions/node/fs/index.ts

@@ -1,8 +1,9 @@
 /**
  * Node's `fs` module: synchronous file and directory operations backed by
- * `runtime/ext_node/fs`. Asynchronous callbacks are not supported (xbintsc has
- * no event loop), so the API is the `*Sync` surface plus `watch`/`watchFile`
- * (which return API-shaped watchers that never emit).
+ * `runtime/ext_node/fs`, plus the `fs.promises` namespace (implemented in
+ * `async_*.c` on top of the asynchronous I/O pool). The callback-style
+ * `readFile`/`writeFile` forms are not supported; `*Sync` and `fs.promises` are.
+ * `watch`/`watchFile` return API-shaped watchers that never emit.
  *
  * `fs.constants` and `fs.promises` are exposed as value getters, so both
  * `import { constants } from "fs"` and the `fs.constants.F_OK` property form
@@ -40,6 +41,14 @@ export const fsModule: NodeModule = {
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/watch.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/constants.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/streams.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_job.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker_dir.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker_fd.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_complete.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_api.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_fh.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/promises.c"),
   ],
   builtins: () => builtins,
   exports: () => exports,

package/src/extensions/node/fs-promises/index.ts

@@ -1,11 +1,11 @@
 /**
  * Node's `fs/promises` module.
  *
- * There is no asynchronous I/O scheduler, so each function wraps the matching
- * synchronous `fs` implementation in an already-settled promise (rejected on
- * failure). `readFile`, `writeFile`, `mkdir`, `readdir`, `stat`, `rm`,
- * `open` (a `FileHandle`), ... are therefore importable from `fs/promises`.
- * `constants` is re-exported as a value getter.
+ * Each function submits a job to the asynchronous I/O pool: the blocking
+ * syscall runs on a worker thread and the promise settles on the event loop, so
+ * awaiting `readFile`, `writeFile`, `stat`, `rm`, `open` (a `FileHandle`), ...
+ * yields to timers and sockets while the I/O is in flight. `constants` is
+ * re-exported as a value getter.
  */
 
 import type { NodeModule } from "../module.js";
@@ -54,7 +54,7 @@ const exports: ModuleExports = {
 export const fsPromisesModule: NodeModule = {
   name: "fs/promises",
   runtimeSources: () => [
-    // Reuse the synchronous fs implementations the promise wrappers delegate to.
+    // Synchronous builtins (used by `watch` and the sync module) plus the async job split.
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/read_file.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/write_file.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/fs_ops.c"),
@@ -63,6 +63,15 @@ export const fsPromisesModule: NodeModule = {
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/link_ops.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/copy_ops.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/constants.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/dir.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/watch.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_job.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker_dir.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_worker_fd.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_complete.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_api.c"),
+    resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/async_fh.c"),
     resolveFrom(import.meta.url, "../../../../runtime/ext_node/fs/promises.c"),
   ],
   builtins: () => builtins,