tranfi 0.1.2 → 0.2.0

package/csrc/main.c

@@ -5,6 +5,7 @@
  *   tranfi 'csv | filter "col(age) > 25" | select name,age | csv'  < in.csv
  *   tranfi -f pipeline.tf < in.csv > out.csv
  *   tranfi -j 'csv | head 5 | csv'   # compile only, output JSON
+ *   tranfi --target sql --dialect duckdb 'csv | head 5 | csv'
  *   tranfi -i input.csv -o output.csv 'csv | filter "col(age) > 25" | csv'
  *
  * Channels:
@@ -14,6 +15,7 @@
 
 #include "tranfi.h"
 #include "internal.h"
+#include "cJSON.h"
 #include "ir.h"
 #include "dsl.h"
 #include "recipes.h"
@@ -22,6 +24,10 @@
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <ctype.h>
+#include <errno.h>
+#include <stdint.h>
+#include <strings.h>
 
 #define READ_BUF_SIZE (64 * 1024)
 #define PULL_BUF_SIZE (64 * 1024)
@@ -41,16 +47,26 @@ static void usage(const char *prog) {
         "  %s 'csv | head 10 | csv'                          # first N rows\n"
         "  %s 'csv | skip 5 | csv'                           # skip first 5 rows\n"
         "  %s 'csv | derive total=col(price)*col(qty) | csv' # computed columns\n"
-        "  %s 'csv | sort age | csv'                         # sort by column\n"
+        "  %s --allow-blocking 'csv | sort age | csv'        # sort known-small input\n"
         "  %s 'csv | unique name | csv'                      # deduplicate\n"
         "  %s 'csv | stats | csv'                            # aggregate stats\n"
         "  %s 'jsonl | filter \"col(x) > 0\" | jsonl'          # JSONL variant\n"
+        "  %s --target sql --dialect duckdb 'csv | head 5 | csv' # print SQL\n"
         "\n"
         "Options:\n"
         "  -f FILE   Read pipeline from file instead of argument\n"
         "  -i FILE   Read input from file instead of stdin\n"
         "  -o FILE   Write output to file instead of stdout\n"
         "  -j        Output plan as JSON (compile only, don't execute)\n"
+        "  --target NAME      Compile target: native, json, or sql\n"
+        "  --dialect NAME     SQL dialect for --target sql (duckdb; sqlite/postgres planned)\n"
+        "  --explain Output target/memory/emit/schema/state execution plan and exit\n"
+        "  --memory max:SIZE  Set memory cap policy, e.g. max:64MB\n"
+        "  --spill-dir DIR    Request disk spill directory for spillable plans\n"
+        "  --engine NAME      Execution engine: native or duckdb\n"
+        "  --stats-json FILE  Write stats side-channel NDJSON to FILE; use - for stderr\n"
+        "  --allow-blocking   Explicitly permit full-input native blocking steps\n"
+        "  --fail-on-blocking  Refuse native execution plans with blocking steps (default)\n"
         "  -p, --progress  Show progress on stderr\n"
         "  -q        Quiet mode (suppress stats on stderr)\n"
         "  --raw     Force raw CSV stats output (disable report formatting)\n"
@@ -63,28 +79,64 @@ static void usage(const char *prog) {
         "  distro, freq, dedup, clean, sample, head, tail, csv2json,\n"
         "  json2csv, tsv2csv, csv2tsv, histogram, hash, samples\n",
         prog, prog, prog, prog, prog, prog, prog, prog,
-        prog, prog, prog, prog, prog, prog);
+        prog, prog, prog, prog, prog, prog, prog);
+}
+
+static int cli_file_output_sink(int channel, const uint8_t *data, size_t len, void *user) {
+    (void)channel;
+    FILE *out = (FILE *)user;
+    if (!out || len == 0) return out ? TF_OK : TF_ERROR;
+    return fwrite(data, 1, len, out) == len ? TF_OK : TF_ERROR;
 }
 
 static char *read_file(const char *path) {
     FILE *f = fopen(path, "r");
     if (!f) return NULL;
 
-    fseek(f, 0, SEEK_END);
+    if (fseek(f, 0, SEEK_END) != 0) { fclose(f); return NULL; }
     long size = ftell(f);
-    fseek(f, 0, SEEK_SET);
+    if (fseek(f, 0, SEEK_SET) != 0) { fclose(f); return NULL; }
 
     if (size <= 0) { fclose(f); return NULL; }
 
-    char *buf = malloc(size + 1);
+    size_t alloc_len;
+    if (tf_size_add((size_t)size, 1, &alloc_len) != TF_OK) { fclose(f); return NULL; }
+    char *buf = tf_mallocarray_checked(alloc_len, sizeof(char));
     if (!buf) { fclose(f); return NULL; }
 
-    size_t nread = fread(buf, 1, size, f);
+    size_t nread = fread(buf, 1, (size_t)size, f);
+    if (ferror(f)) {
+        free(buf);
+        fclose(f);
+        return NULL;
+    }
     fclose(f);
     buf[nread] = '\0';
     return buf;
 }
 
+typedef enum {
+    CLI_ENGINE_NATIVE,
+    CLI_ENGINE_DUCKDB,
+} cli_engine;
+
+typedef struct {
+    cli_engine engine;
+    int allow_blocking;
+    int fail_on_blocking_seen;
+    int has_memory_limit;
+    size_t memory_limit_bytes;
+    const char *spill_dir;
+} cli_memory_policy;
+
+static const char *engine_name(cli_engine engine) {
+    switch (engine) {
+        case CLI_ENGINE_NATIVE: return "native";
+        case CLI_ENGINE_DUCKDB: return "duckdb";
+        default: return "unknown";
+    }
+}
+
 static const char *format_bytes(size_t bytes, char *buf, size_t buf_size) {
     if (bytes < 1024) {
         snprintf(buf, buf_size, "%zuB", bytes);
@@ -98,15 +150,601 @@ static const char *format_bytes(size_t bytes, char *buf, size_t buf_size) {
     return buf;
 }
 
+static int parse_size_bytes(const char *text, size_t *out, char *err, size_t err_size) {
+    const char *p = text;
+    if (!p || !*p) {
+        snprintf(err, err_size, "empty size");
+        return -1;
+    }
+    if (strncmp(p, "max:", 4) == 0 || strncmp(p, "max=", 4) == 0) p += 4;
+    while (isspace((unsigned char)*p)) p++;
+    if (!isdigit((unsigned char)*p)) {
+        snprintf(err, err_size, "expected size such as 64MB or max:64MB");
+        return -1;
+    }
+
+    errno = 0;
+    char *end = NULL;
+    unsigned long long value = strtoull(p, &end, 10);
+    if (errno != 0 || end == p || value == 0) {
+        snprintf(err, err_size, "invalid positive size '%s'", text);
+        return -1;
+    }
+    while (isspace((unsigned char)*end)) end++;
+
+    char suffix[8] = {0};
+    size_t si = 0;
+    while (*end && si + 1 < sizeof(suffix)) {
+        suffix[si++] = (char)tolower((unsigned char)*end++);
+    }
+    while (isspace((unsigned char)*end)) end++;
+    if (*end) {
+        snprintf(err, err_size, "invalid size suffix in '%s'", text);
+        return -1;
+    }
+
+    unsigned long long mul = 1;
+    if (suffix[0] == '\0' || strcmp(suffix, "b") == 0) mul = 1;
+    else if (strcmp(suffix, "k") == 0 || strcmp(suffix, "kb") == 0 || strcmp(suffix, "kib") == 0) mul = 1024ULL;
+    else if (strcmp(suffix, "m") == 0 || strcmp(suffix, "mb") == 0 || strcmp(suffix, "mib") == 0) mul = 1024ULL * 1024ULL;
+    else if (strcmp(suffix, "g") == 0 || strcmp(suffix, "gb") == 0 || strcmp(suffix, "gib") == 0) mul = 1024ULL * 1024ULL * 1024ULL;
+    else if (strcmp(suffix, "t") == 0 || strcmp(suffix, "tb") == 0 || strcmp(suffix, "tib") == 0) mul = 1024ULL * 1024ULL * 1024ULL * 1024ULL;
+    else {
+        snprintf(err, err_size, "unsupported size suffix '%s'", suffix);
+        return -1;
+    }
+
+    if (value > (unsigned long long)SIZE_MAX / mul) {
+        snprintf(err, err_size, "size '%s' is too large", text);
+        return -1;
+    }
+    *out = (size_t)(value * mul);
+    return 0;
+}
+
+static int set_engine(cli_memory_policy *policy, const char *name, char *err, size_t err_size) {
+    if (strcmp(name, "native") == 0) {
+        policy->engine = CLI_ENGINE_NATIVE;
+        return 0;
+    }
+    if (strcmp(name, "duckdb") == 0 || strcmp(name, "sql") == 0) {
+        policy->engine = CLI_ENGINE_DUCKDB;
+        return 0;
+    }
+    snprintf(err, err_size, "unknown engine '%s' (expected native or duckdb)", name);
+    return -1;
+}
+
+
+static int is_known_sql_dialect(const char *name) {
+    return name && (strcmp(name, "duckdb") == 0 ||
+                    strcmp(name, "sqlite") == 0 ||
+                    strcmp(name, "postgres") == 0);
+}
+
+static int sql_dialect_supported(const char *name) {
+    return name && strcmp(name, "duckdb") == 0;
+}
+
+static int memory_rank(tf_memory_class cls) {
+    switch (cls) {
+        case TF_MEM_ROW_LOCAL: return 0;
+        case TF_MEM_BOUNDED_STATE: return 1;
+        case TF_MEM_KEY_STATE: return 2;
+        case TF_MEM_EXTERNAL: return 3;
+        case TF_MEM_BLOCKING: return 4;
+        default: return 5;
+    }
+}
+
+static void print_cap_item(FILE *f, int *first, const char *name) {
+    fprintf(f, "%s%s", *first ? "" : ",", name);
+    *first = 0;
+}
+
+
+typedef struct {
+    char *data;
+    size_t len;
+    size_t cap;
+} cli_string_builder;
+
+static int sb_reserve(cli_string_builder *sb, size_t extra) {
+    if (!sb) return -1;
+    size_t need;
+    if (tf_size_add(sb->len, extra, &need) != TF_OK ||
+        tf_size_add(need, 1, &need) != TF_OK) {
+        return -1;
+    }
+    if (need <= sb->cap) return 0;
+    size_t new_cap;
+    if (tf_size_grow_pow2(sb->cap, need, 256, &new_cap) != TF_OK) return -1;
+    char *tmp = tf_reallocarray_checked(sb->data, new_cap, sizeof(char));
+    if (!tmp) return -1;
+    sb->data = tmp;
+    sb->cap = new_cap;
+    return 0;
+}
+
+static int sb_append_n(cli_string_builder *sb, const char *s, size_t n) {
+    if (!s) return 0;
+    if (sb_reserve(sb, n) != 0) return -1;
+    memcpy(sb->data + sb->len, s, n);
+    sb->len += n;
+    sb->data[sb->len] = '\0';
+    return 0;
+}
+
+static int sb_append(cli_string_builder *sb, const char *s) {
+    return sb_append_n(sb, s, s ? strlen(s) : 0);
+}
+
+static int sb_append_char(cli_string_builder *sb, char ch) {
+    if (sb_reserve(sb, 1) != 0) return -1;
+    sb->data[sb->len++] = ch;
+    sb->data[sb->len] = '\0';
+    return 0;
+}
+
+static int cli_report_buffer_append(char **buf, size_t *len, size_t *cap,
+                                    const uint8_t *data, size_t n) {
+    if (!buf || !len || !cap || !*buf || (!data && n > 0)) return -1;
+    size_t need;
+    if (tf_size_add(*len, n, &need) != TF_OK) return -1;
+    if (need > *cap) {
+        size_t new_cap;
+        if (tf_size_grow_pow2(*cap, need, PULL_BUF_SIZE, &new_cap) != TF_OK) return -1;
+        char *tmp = tf_reallocarray_checked(*buf, new_cap, sizeof(char));
+        if (!tmp) return -1;
+        *buf = tmp;
+        *cap = new_cap;
+    }
+    if (n > 0) memcpy(*buf + *len, data, n);
+    *len = need;
+    return 0;
+}
+
+static void cli_disable_report_buffer(FILE *fout, char **buf, size_t *len, size_t *cap) {
+    if (!buf || !len || !cap) return;
+    if (fout && *buf && *len > 0) fwrite(*buf, 1, *len, fout);
+    free(*buf);
+    *buf = NULL;
+    *len = 0;
+    *cap = 0;
+}
+
+static int cli_is_bare_dsl_token(const char *s) {
+    if (!s || !*s) return 0;
+    for (const unsigned char *p = (const unsigned char *)s; *p; p++) {
+        if (isspace(*p) || *p == '|' || *p == '=' || *p == '"' || *p == '\'' || *p == '\\') return 0;
+    }
+    return 1;
+}
+
+static const char *cli_normalized_step_name(const char *op) {
+    if (!op) return "unknown";
+    if (strcmp(op, "codec.csv.decode") == 0 || strcmp(op, "codec.csv.encode") == 0) return "csv";
+    if (strcmp(op, "codec.jsonl.decode") == 0 || strcmp(op, "codec.jsonl.encode") == 0) return "jsonl";
+    if (strcmp(op, "codec.text.decode") == 0 || strcmp(op, "codec.text.encode") == 0) return "text";
+    if (strcmp(op, "codec.table.encode") == 0) return "table";
+    return op;
+}
+
+static int cli_append_json_value_token(cli_string_builder *sb, const cJSON *value) {
+    if (!value) return sb_append(sb, "null");
+    if (cJSON_IsString(value) && value->valuestring && cli_is_bare_dsl_token(value->valuestring)) {
+        return sb_append(sb, value->valuestring);
+    }
+    char *json = cJSON_PrintUnformatted((cJSON *)value);
+    if (!json) return -1;
+    int rc = sb_append(sb, json);
+    free(json);
+    return rc;
+}
+
+static int cli_append_normalized_arg(cli_string_builder *sb, const cJSON *arg) {
+    if (!arg || !arg->string) return 0;
+    if (sb_append_char(sb, ' ') != 0) return -1;
+    if (sb_append(sb, arg->string) != 0) return -1;
+    if (sb_append_char(sb, '=') != 0) return -1;
+    return cli_append_json_value_token(sb, arg);
+}
+
+static char *cli_ir_to_normalized_dsl(const tf_ir_plan *ir) {
+    if (!ir) return NULL;
+    cli_string_builder sb = {0};
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        const tf_ir_node *node = &ir->nodes[i];
+        if (i > 0 && sb_append(&sb, " | ") != 0) goto fail;
+        if (sb_append(&sb, cli_normalized_step_name(node->op)) != 0) goto fail;
+        if (node->args && cJSON_IsObject(node->args)) {
+            for (const cJSON *arg = node->args->child; arg; arg = arg->next) {
+                if (arg->string &&
+                    (strcmp(arg->string, TF_POLICY_VALIDATED_FILE_PATH_ARG) == 0 ||
+                     strcmp(arg->string, TF_POLICY_VALIDATED_RULES_FILE_PATH_ARG) == 0)) {
+                    continue;
+                }
+                if (cli_append_normalized_arg(&sb, arg) != 0) goto fail;
+            }
+        }
+    }
+    if (!sb.data) {
+        sb.data = strdup("");
+        if (!sb.data) return NULL;
+    }
+    return sb.data;
+
+fail:
+    free(sb.data);
+    return NULL;
+}
+
+static void print_caps(FILE *f, uint32_t caps) {
+    int first = 1;
+    if (caps & TF_CAP_STREAMING) print_cap_item(f, &first, "streaming");
+    if (caps & TF_CAP_BOUNDED_MEMORY) print_cap_item(f, &first, "bounded_memory");
+    if (caps & TF_CAP_BROWSER_SAFE) print_cap_item(f, &first, "browser_safe");
+    if (caps & TF_CAP_DETERMINISTIC) print_cap_item(f, &first, "deterministic");
+    if (caps & TF_CAP_FS) print_cap_item(f, &first, "fs");
+    if (caps & TF_CAP_NET) print_cap_item(f, &first, "net");
+    if (first) fprintf(f, "none");
+}
+
+static const tf_ir_node *first_node_with_memory_class(const tf_ir_plan *ir, tf_memory_class cls) {
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        if (ir->nodes[i].memory_class == cls) return &ir->nodes[i];
+    }
+    return NULL;
+}
+
+static const tf_ir_node *first_blocking_node(const tf_ir_plan *ir) {
+    return first_node_with_memory_class(ir, TF_MEM_BLOCKING);
+}
+
+static const tf_ir_node *first_key_state_node(const tf_ir_plan *ir) {
+    return first_node_with_memory_class(ir, TF_MEM_KEY_STATE);
+}
+
+static int has_sort_head_pattern(const tf_ir_plan *ir) {
+    for (size_t i = 0; i + 1 < ir->n_nodes; i++) {
+        if (strcmp(ir->nodes[i].op, "sort") == 0 && strcmp(ir->nodes[i + 1].op, "head") == 0)
+            return 1;
+    }
+    return 0;
+}
+
+static int node_arg_true(const tf_ir_node *node, const char *name) {
+    cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
+    return cJSON_IsTrue(item);
+}
+
+static int node_op_is_unique(const tf_ir_node *node) {
+    return node && node->op &&
+           (strcmp(node->op, "unique") == 0 || strcmp(node->op, "dedup") == 0);
+}
+
+static int node_array_arg_nonempty_main(const tf_ir_node *node, const char *name) {
+    cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
+    return cJSON_IsArray(item) && cJSON_GetArraySize(item) > 0;
+}
+
+static int node_positive_arg_main(const tf_ir_node *node, const char *name) {
+    cJSON *item = node && node->args ? cJSON_GetObjectItemCaseSensitive(node->args, name) : NULL;
+    return cJSON_IsNumber(item) && item->valuedouble > 0.0;
+}
+
+static int node_op_is_pivot_spillable(const tf_ir_node *node) {
+    return node && node->op && strcmp(node->op, "pivot") == 0 &&
+           !node_arg_true(node, "sorted") &&
+           (node_array_arg_nonempty_main(node, "categories") || node_positive_arg_main(node, "max_categories"));
+}
+
+
+static int node_op_is_join_spillable(const tf_ir_node *node) {
+    if (!node || !node->op) return 0;
+    if (strcmp(node->op, "semi-join") == 0 || strcmp(node->op, "anti-join") == 0) return 1;
+    if (strcmp(node->op, "join") != 0) return 0;
+    if (!node->args) return 1;
+    cJSON *how = cJSON_GetObjectItemCaseSensitive(node->args, "how");
+    if (!cJSON_IsString(how) || !how->valuestring) return 1;
+    return strcmp(how->valuestring, "inner") == 0 || strcmp(how->valuestring, "left") == 0 ||
+           strcmp(how->valuestring, "semi") == 0 || strcmp(how->valuestring, "anti") == 0;
+}
+
+static int node_op_is_row_set_spillable(const tf_ir_node *node) {
+    return node && node->op &&
+           (strcmp(node->op, "intersect") == 0 || strcmp(node->op, "setdiff") == 0 ||
+            strcmp(node->op, "intersect-all") == 0 || strcmp(node->op, "setdiff-all") == 0 ||
+            strcmp(node->op, "union") == 0);
+}
+
+static int blocking_node_has_native_spill(const tf_ir_node *node) {
+    return node && node->op && (strcmp(node->op, "sort") == 0 || node_op_is_pivot_spillable(node));
+}
+
+static int key_state_node_has_native_spill(const tf_ir_node *node) {
+    return ((node_op_is_unique(node) ||
+             (node && node->op && strcmp(node->op, "group-agg") == 0) ||
+             node_op_is_join_spillable(node) ||
+             node_op_is_row_set_spillable(node)) &&
+            !node_arg_true(node, "sorted"));
+}
+
+static const tf_ir_node *first_key_state_node_without_native_spill(const tf_ir_plan *ir) {
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        const tf_ir_node *node = &ir->nodes[i];
+        if (node->memory_class == TF_MEM_KEY_STATE && !key_state_node_has_native_spill(node))
+            return node;
+    }
+    return NULL;
+}
+
+static int node_uses_native_spill(const tf_ir_node *node) {
+    return blocking_node_has_native_spill(node) || key_state_node_has_native_spill(node);
+}
+
+static const char *explain_step_target(const tf_ir_node *node, const cli_memory_policy *policy) {
+    if (policy->engine == CLI_ENGINE_DUCKDB) return "duckdb_sql";
+    if (policy->engine == CLI_ENGINE_NATIVE && policy->spill_dir && node_uses_native_spill(node))
+        return "native_spill";
+    return "native";
+}
+
+static const tf_ir_node *first_blocking_node_without_native_spill(const tf_ir_plan *ir) {
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        const tf_ir_node *node = &ir->nodes[i];
+        if (node->memory_class == TF_MEM_BLOCKING && !blocking_node_has_native_spill(node))
+            return node;
+    }
+    return NULL;
+}
+
+static int blocking_plan_can_use_native_spill(const tf_ir_plan *ir) {
+    return first_blocking_node(ir) && !first_blocking_node_without_native_spill(ir);
+}
+
+static const char *native_spill_support_summary(void) {
+    return "native spill currently supports sort, capped unsorted pivot "
+           "(categories=... or max_categories=N), unsorted unique/dedup, "
+           "unsorted group-agg, capped unsorted inner/left joins, unsorted "
+           "semi/anti filtering joins, unsorted intersect/setdiff/"
+           "intersect-all/setdiff-all, and duplicate-eliminating union";
+}
+
+static int set_json_item(cJSON *obj, const char *name, cJSON *item) {
+    if (!obj || !name || !item) {
+        cJSON_Delete(item);
+        return -1;
+    }
+    if (cJSON_GetObjectItemCaseSensitive(obj, name)) {
+        if (cJSON_ReplaceItemInObjectCaseSensitive(obj, name, item)) return 0;
+        cJSON_Delete(item);
+        return -1;
+    }
+    if (tf_json_add_item(obj, name, item) != TF_OK) {
+        cJSON_Delete(item);
+        return -1;
+    }
+    return 0;
+}
+
+static int set_json_string(cJSON *obj, const char *name, const char *value) {
+    return set_json_item(obj, name, cJSON_CreateString(value ? value : ""));
+}
+
+static int set_json_number(cJSON *obj, const char *name, double value) {
+    return set_json_item(obj, name, cJSON_CreateNumber(value));
+}
+
+static int apply_native_spill_policy(tf_ir_plan *ir, const cli_memory_policy *policy) {
+    if (!policy->spill_dir || policy->engine != CLI_ENGINE_NATIVE) return 0;
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        tf_ir_node *node = &ir->nodes[i];
+        if (!node_uses_native_spill(node)) continue;
+        if (!node->args) {
+            node->args = cJSON_CreateObject();
+            if (!node->args) return -1;
+        }
+        if (set_json_string(node->args, "spill_dir", policy->spill_dir) != 0) return -1;
+        if (policy->has_memory_limit &&
+            set_json_number(node->args, "spill_memory_bytes", (double)policy->memory_limit_bytes) != 0)
+            return -1;
+    }
+    return 0;
+}
+
+static int validate_memory_policy(const tf_ir_plan *ir, const cli_memory_policy *policy) {
+    const tf_ir_node *blocking = first_blocking_node(ir);
+    const tf_ir_node *key_state = first_key_state_node(ir);
+
+    if (policy->engine == CLI_ENGINE_DUCKDB) return 0;
+
+    if (policy->spill_dir) {
+        const tf_ir_node *unsupported_key = first_key_state_node_without_native_spill(ir);
+        if (unsupported_key) {
+            fprintf(stderr,
+                    "error: --spill-dir was requested, but native spill is not implemented yet for key-state step '%s'\n",
+                    unsupported_key->op);
+            fprintf(stderr,
+                    "hint: %s; use op-specific caps or an external engine for this plan\n",
+                    native_spill_support_summary());
+            return 1;
+        }
+        const tf_ir_node *unsupported = first_blocking_node_without_native_spill(ir);
+        if (unsupported) {
+            fprintf(stderr,
+                    "error: --spill-dir was requested, but native spill is unavailable for blocking step '%s' with the current arguments\n",
+                    unsupported->op);
+            fprintf(stderr,
+                    "hint: %s; use --allow-blocking only for known-small data or an external engine for this plan\n",
+                    native_spill_support_summary());
+            return 1;
+        }
+    }
+
+    if (policy->has_memory_limit && blocking &&
+        !(policy->spill_dir && blocking_plan_can_use_native_spill(ir))) {
+        char cap[32];
+        fprintf(stderr,
+                "error: --memory %s was requested, but native byte caps are not implemented for blocking step '%s' (state=%s)\n",
+                format_bytes(policy->memory_limit_bytes, cap, sizeof(cap)),
+                blocking->op,
+                blocking->state_estimate ? blocking->state_estimate : "unknown");
+        fprintf(stderr,
+                "hint: rewrite to a bounded op, choose an external engine, or use --spill-dir when the step has a native spill mode and required caps\n");
+        return 1;
+    }
+
+    if (policy->has_memory_limit && key_state) {
+        size_t estimated = 0;
+        const tf_ir_node *failed = NULL;
+        char reason[192] = {0};
+        if (!tf_estimate_key_state_plan_bytes(ir, &estimated, &failed, reason, sizeof(reason))) {
+            char cap[32];
+            fprintf(stderr,
+                    "error: --memory %s was requested, but %s\n",
+                    format_bytes(policy->memory_limit_bytes, cap, sizeof(cap)),
+                    reason[0] ? reason : "a key-state step is not byte-bounded");
+            if (failed) {
+                fprintf(stderr,
+                        "hint: add the op-specific cap for step '%s' or use an external/spill engine\n",
+                        failed->op);
+            }
+            return 1;
+        }
+        if (estimated > policy->memory_limit_bytes) {
+            char cap[32], est[32];
+            fprintf(stderr,
+                    "error: estimated native key-state memory %s exceeds --memory %s\n",
+                    format_bytes(estimated, est, sizeof(est)),
+                    format_bytes(policy->memory_limit_bytes, cap, sizeof(cap)));
+            fprintf(stderr,
+                    "hint: lower key/category/lookup caps or raise --memory for this known-bounded plan\n");
+            return 1;
+        }
+    }
+
+    if (blocking && !policy->allow_blocking &&
+        !(policy->spill_dir && blocking_plan_can_use_native_spill(ir))) {
+        fprintf(stderr,
+                "error: blocking step '%s' requires full input in native mode (state=%s)\n",
+                blocking->op,
+                blocking->state_estimate ? blocking->state_estimate : "unknown");
+        fprintf(stderr,
+                "hint: add --allow-blocking only for known-small inputs, rewrite to a bounded op, or choose an external engine/spill mode when available\n");
+        if (has_sort_head_pattern(ir)) {
+            fprintf(stderr,
+                    "hint: replace sort | head N with top N column when top-k semantics are acceptable\n");
+        }
+        return 1;
+    }
+
+    return 0;
+}
+
+static void print_explain(const tf_ir_plan *ir, const cli_memory_policy *policy) {
+    tf_memory_class worst_mem = TF_MEM_ROW_LOCAL;
+    const char *worst_state = "O(batch_rows * columns)";
+    int has_flush = 0;
+    int has_per_batch = 0;
+    int has_data_schema = 0;
+
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        const tf_ir_node *node = &ir->nodes[i];
+        if (memory_rank(node->memory_class) > memory_rank(worst_mem)) {
+            worst_mem = node->memory_class;
+            worst_state = node->state_estimate ? node->state_estimate : "unknown";
+        }
+        if (node->emit_class == TF_EMIT_ON_FLUSH) has_flush = 1;
+        if (node->emit_class == TF_EMIT_PER_BATCH) has_per_batch = 1;
+        if (node->schema_class == TF_SCHEMA_DATA_DEPENDENT) has_data_schema = 1;
+    }
+
+    char cap[32];
+    printf("Tranfi execution plan\n");
+    char *normalized_dsl = cli_ir_to_normalized_dsl(ir);
+    printf("normalized_dsl: %s\n", normalized_dsl ? normalized_dsl : "null");
+    free(normalized_dsl);
+    printf("execution_target: %s\n", policy->spill_dir && policy->engine == CLI_ENGINE_NATIVE ? "native+spill" : engine_name(policy->engine));
+    printf("memory_policy: %s\n", policy->spill_dir ? "spill" : (policy->allow_blocking ? "allow_blocking" : "strict"));
+    printf("memory_limit: %s\n", policy->has_memory_limit ? format_bytes(policy->memory_limit_bytes, cap, sizeof(cap)) : "none");
+    printf("spill_dir: %s\n", policy->spill_dir ? policy->spill_dir : "none");
+    printf("memory_class: %s\n", tf_memory_class_name(worst_mem));
+    printf("emit_class: %s\n", has_flush && has_per_batch ? "mixed" :
+           (has_flush ? "on_flush" : "per_batch"));
+    printf("schema_class: %s\n", has_data_schema ? "data_dependent" : "stable_or_parametric");
+    printf("state_estimate: %s\n", worst_state);
+    if (first_key_state_node(ir)) {
+        size_t key_bytes = 0;
+        const tf_ir_node *failed = NULL;
+        char reason[192] = {0};
+        if (tf_estimate_key_state_plan_bytes(ir, &key_bytes, &failed, reason, sizeof(reason))) {
+            char key_est[32];
+            printf("state_bytes_estimate: %s\n", format_bytes(key_bytes, key_est, sizeof(key_est)));
+        } else {
+            printf("state_bytes_estimate: unbounded (%s)\n", reason[0] ? reason : "missing key-state byte estimator");
+        }
+    } else {
+        printf("state_bytes_estimate: none\n");
+    }
+    if (first_blocking_node(ir)) {
+        if (policy->spill_dir && policy->engine == CLI_ENGINE_NATIVE && blocking_plan_can_use_native_spill(ir))
+            printf("note: blocking step will use native spill files\n");
+        else if (policy->engine == CLI_ENGINE_NATIVE && !policy->allow_blocking)
+            printf("warning: blocking native step present; default execution will reject it unless --allow-blocking is set\n");
+        else
+            printf("warning: blocking step present; selected policy must provide a bounded/external execution path\n");
+        if (has_sort_head_pattern(ir))
+            printf("hint: replace sort | head N with top N column when top-k semantics are acceptable\n");
+    }
+    printf("steps:\n");
+    for (size_t i = 0; i < ir->n_nodes; i++) {
+        const tf_ir_node *node = &ir->nodes[i];
+        printf("  %zu. %s\n", i, node->op);
+        printf("     target=%s memory=%s emit=%s schema=%s state=%s caps=",
+               explain_step_target(node, policy),
+               tf_memory_class_name(node->memory_class),
+               tf_emit_class_name(node->emit_class),
+               tf_schema_class_name(node->schema_class),
+               node->state_estimate ? node->state_estimate : "unknown");
+        print_caps(stdout, node->caps);
+        printf("\n");
+        if (node->memory_class == TF_MEM_KEY_STATE) {
+            size_t step_bytes = 0;
+            char reason[192] = {0};
+            if (tf_estimate_step_state_bytes(node, &step_bytes, reason, sizeof(reason))) {
+                char step_est[32];
+                printf("     state_bytes_estimate=%s\n", format_bytes(step_bytes, step_est, sizeof(step_est)));
+            } else {
+                printf("     state_bytes_estimate=unbounded (%s)\n", reason[0] ? reason : "missing key-state byte estimator");
+            }
+        }
+    }
+
+    char *ir_json = tf_ir_to_json(ir);
+    if (ir_json) {
+        printf("ir_json: %s\n", ir_json);
+        free(ir_json);
+    } else {
+        printf("ir_json: null\n");
+    }
+}
+
 int main(int argc, char **argv) {
     const char *pipeline_file = NULL;
     const char *pipeline_text = NULL;
     const char *input_file = NULL;
     const char *output_file = NULL;
     int json_mode = 0;
+    int sql_mode = 0;
+    int explain_mode = 0;
+    const char *sql_dialect = "duckdb";
+    cli_memory_policy policy = {0};
+    policy.engine = CLI_ENGINE_NATIVE;
     int quiet = 0;
     int progress = 0;
     int raw_stats = 0;
+    const char *stats_json_file = NULL;
 
     /* Parse options */
     int argi = 1;
@@ -129,12 +767,126 @@ int main(int argc, char **argv) {
             return 0;
         } else if (strcmp(opt, "-j") == 0) {
             json_mode = 1;
+            sql_mode = 0;
+        } else if (strcmp(opt, "--target") == 0 || strncmp(opt, "--target=", 9) == 0) {
+            const char *value = NULL;
+            if (strncmp(opt, "--target=", 9) == 0) {
+                value = opt + 9;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --target requires native, json, or sql\n");
+                    return 1;
+                }
+                value = argv[argi];
+            }
+            if (strcmp(value, "native") == 0) {
+                json_mode = 0;
+                sql_mode = 0;
+            } else if (strcmp(value, "json") == 0 || strcmp(value, "ir") == 0) {
+                json_mode = 1;
+                sql_mode = 0;
+            } else if (strcmp(value, "sql") == 0) {
+                json_mode = 0;
+                sql_mode = 1;
+            } else {
+                fprintf(stderr, "error: unknown --target '%s' (expected native, json, or sql)\n", value);
+                return 1;
+            }
+        } else if (strcmp(opt, "--dialect") == 0 || strncmp(opt, "--dialect=", 10) == 0) {
+            const char *value = NULL;
+            if (strncmp(opt, "--dialect=", 10) == 0) {
+                value = opt + 10;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --dialect requires duckdb, sqlite, or postgres\n");
+                    return 1;
+                }
+                value = argv[argi];
+            }
+            if (!is_known_sql_dialect(value)) {
+                fprintf(stderr, "error: unknown SQL dialect '%s' (expected duckdb, sqlite, or postgres)\n", value);
+                return 1;
+            }
+            sql_dialect = value;
+        } else if (strcmp(opt, "--explain") == 0) {
+            explain_mode = 1;
+        } else if (strcmp(opt, "--allow-blocking") == 0) {
+            policy.allow_blocking = 1;
+        } else if (strcmp(opt, "--fail-on-blocking") == 0) {
+            policy.fail_on_blocking_seen = 1;
+        } else if (strcmp(opt, "--memory") == 0 || strncmp(opt, "--memory=", 9) == 0) {
+            const char *value = NULL;
+            if (strncmp(opt, "--memory=", 9) == 0) {
+                value = opt + 9;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --memory requires a size such as max:64MB\n");
+                    return 1;
+                }
+                value = argv[argi];
+            }
+            char err[160];
+            if (parse_size_bytes(value, &policy.memory_limit_bytes, err, sizeof(err)) != 0) {
+                fprintf(stderr, "error: invalid --memory value: %s\n", err);
+                return 1;
+            }
+            policy.has_memory_limit = 1;
+        } else if (strcmp(opt, "--spill-dir") == 0 || strncmp(opt, "--spill-dir=", 12) == 0) {
+            if (strncmp(opt, "--spill-dir=", 12) == 0) {
+                policy.spill_dir = opt + 12;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --spill-dir requires a directory argument\n");
+                    return 1;
+                }
+                policy.spill_dir = argv[argi];
+            }
+            if (!policy.spill_dir || policy.spill_dir[0] == '\0') {
+                fprintf(stderr, "error: --spill-dir requires a non-empty directory\n");
+                return 1;
+            }
+        } else if (strcmp(opt, "--engine") == 0 || strncmp(opt, "--engine=", 9) == 0) {
+            const char *value = NULL;
+            if (strncmp(opt, "--engine=", 9) == 0) {
+                value = opt + 9;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --engine requires native or duckdb\n");
+                    return 1;
+                }
+                value = argv[argi];
+            }
+            char err[160];
+            if (set_engine(&policy, value, err, sizeof(err)) != 0) {
+                fprintf(stderr, "error: %s\n", err);
+                return 1;
+            }
         } else if (strcmp(opt, "-q") == 0) {
             quiet = 1;
         } else if (strcmp(opt, "-p") == 0 || strcmp(opt, "--progress") == 0) {
             progress = 1;
         } else if (strcmp(opt, "--raw") == 0) {
             raw_stats = 1;
+        } else if (strcmp(opt, "--stats-json") == 0 || strncmp(opt, "--stats-json=", 13) == 0) {
+            if (strncmp(opt, "--stats-json=", 13) == 0) {
+                stats_json_file = opt + 13;
+            } else {
+                argi++;
+                if (argi >= argc) {
+                    fprintf(stderr, "error: --stats-json requires a file path, or - for stderr\n");
+                    return 1;
+                }
+                stats_json_file = argv[argi];
+            }
+            if (!stats_json_file || stats_json_file[0] == '\0') {
+                fprintf(stderr, "error: --stats-json requires a non-empty file path, or - for stderr\n");
+                return 1;
+            }
         } else if (strcmp(opt, "-f") == 0) {
             argi++;
             if (argi >= argc) {
@@ -163,6 +915,14 @@ int main(int argc, char **argv) {
         argi++;
     }
 
+    if (policy.allow_blocking && policy.fail_on_blocking_seen) {
+        fprintf(stderr, "error: --allow-blocking conflicts with --fail-on-blocking\n");
+        return 1;
+    }
+    if (sql_mode) {
+        policy.engine = CLI_ENGINE_DUCKDB;
+    }
+
     /* Get pipeline text */
     char *file_content = NULL;
     if (pipeline_file) {
@@ -218,8 +978,58 @@ int main(int argc, char **argv) {
         return 1;
     }
 
-    /* Schema inference (best-effort) */
-    tf_ir_infer_schema(ir);
+    /* Schema inference allows unknown runtime schemas but reports invalid known-schema plans. */
+    tf_set_last_error(NULL);
+    if (tf_ir_infer_schema(ir) != TF_OK) {
+        const char *detail = tf_last_error();
+        if (detail && detail[0])
+            fprintf(stderr, "error: schema inference failed: %s\n", detail);
+        else
+            fprintf(stderr, "error: schema inference failed\n");
+        tf_ir_plan_free(ir);
+        return 1;
+    }
+
+    if (apply_native_spill_policy(ir, &policy) != 0) {
+        fprintf(stderr, "error: failed to apply native spill policy\n");
+        tf_ir_plan_free(ir);
+        return 1;
+    }
+
+    if (tf_ir_validate(ir) != TF_OK) {
+        fprintf(stderr, "error: %s\n", ir->error ? ir->error : "validation failed after applying native spill policy");
+        tf_ir_plan_free(ir);
+        return 1;
+    }
+
+    if (explain_mode) {
+        print_explain(ir, &policy);
+        tf_ir_plan_free(ir);
+        return 0;
+    }
+
+    if (sql_mode) {
+        if (!sql_dialect_supported(sql_dialect)) {
+            fprintf(stderr,
+                    "error: SQL dialect '%s' is recognized but not implemented yet; only duckdb lowering is available\n",
+                    sql_dialect);
+            tf_ir_plan_free(ir);
+            return 1;
+        }
+        char *sql_error = NULL;
+        char *sql = tf_ir_plan_to_sql(ir, &sql_error);
+        if (!sql) {
+            fprintf(stderr, "error: SQL compile failed for dialect '%s': %s\n",
+                    sql_dialect, sql_error ? sql_error : "unknown error");
+            free(sql_error);
+            tf_ir_plan_free(ir);
+            return 1;
+        }
+        printf("%s\n", sql);
+        tf_string_free(sql);
+        tf_ir_plan_free(ir);
+        return 0;
+    }
 
     /* JSON mode: print IR and exit */
     if (json_mode) {
@@ -232,6 +1042,18 @@ int main(int argc, char **argv) {
         return 0;
     }
 
+    if (validate_memory_policy(ir, &policy) != 0) {
+        tf_ir_plan_free(ir);
+        return 1;
+    }
+
+    if (policy.engine == CLI_ENGINE_DUCKDB) {
+        fprintf(stderr,
+                "error: the C CLI cannot execute --engine duckdb directly; use Python/Node with engine='duckdb' or compile SQL with --target sql --dialect duckdb\n");
+        tf_ir_plan_free(ir);
+        return 1;
+    }
+
     /* Compile to native pipeline */
     tf_pipeline *p = tf_pipeline_create_from_ir(ir);
     tf_ir_plan_free(ir);
@@ -245,6 +1067,8 @@ int main(int argc, char **argv) {
     /* Open I/O files */
     FILE *fin = stdin;
     FILE *fout = stdout;
+    FILE *fstats = NULL;
+    int close_stats = 0;
 
     if (input_file) {
         fin = fopen(input_file, "rb");
@@ -265,11 +1089,37 @@ int main(int argc, char **argv) {
         }
     }
 
+    if (stats_json_file) {
+        if (strcmp(stats_json_file, "-") == 0) {
+            fstats = stderr;
+        } else {
+            fstats = fopen(stats_json_file, "wb");
+            if (!fstats) {
+                fprintf(stderr, "error: cannot open stats JSON file '%s'\n", stats_json_file);
+                if (fin != stdin) fclose(fin);
+                if (fout != stdout) fclose(fout);
+                tf_pipeline_free(p);
+                return 1;
+            }
+            close_stats = 1;
+        }
+    }
+
     /* Decide whether to buffer output for report formatting.
      * When stdout is a TTY and --raw is not set, buffer main output
      * and try to render it as a rich report. Falls back to raw CSV
      * if the output doesn't look like a stats table. */
     int try_report = !raw_stats && !output_file && isatty(STDOUT_FILENO);
+    if (!try_report) {
+        if (tf_pipeline_set_sink(p, TF_CHAN_MAIN, cli_file_output_sink, fout) != TF_OK) {
+            fprintf(stderr, "error: failed to attach output sink\n");
+            if (fin != stdin) fclose(fin);
+            if (fout != stdout) fclose(fout);
+            if (close_stats && fstats) fclose(fstats);
+            tf_pipeline_free(p);
+            return 1;
+        }
+    }
 
     /* Stream input → pipeline → output */
     uint8_t read_buf[READ_BUF_SIZE];
@@ -279,7 +1129,11 @@ int main(int argc, char **argv) {
     /* Output buffer (used when try_report is true) */
     size_t out_cap = PULL_BUF_SIZE;
     size_t out_len = 0;
-    char *out_buf = try_report ? malloc(out_cap) : NULL;
+    char *out_buf = try_report ? tf_mallocarray_checked(out_cap, sizeof(char)) : NULL;
+    if (try_report && !out_buf) {
+        out_cap = 0;
+        try_report = 0;
+    }
 
     while ((nread = fread(read_buf, 1, sizeof(read_buf), fin)) > 0) {
         if (tf_pipeline_push(p, read_buf, nread) != TF_OK) {
@@ -288,6 +1142,7 @@ int main(int argc, char **argv) {
             free(out_buf);
             if (fin != stdin) fclose(fin);
             if (fout != stdout) fclose(fout);
+            if (close_stats && fstats) fclose(fstats);
             tf_pipeline_free(p);
             return 1;
         }
@@ -299,15 +1154,10 @@ int main(int argc, char **argv) {
         size_t n;
         while ((n = tf_pipeline_pull(p, TF_CHAN_MAIN, pull_buf, sizeof(pull_buf))) > 0) {
             if (try_report && out_buf) {
-                while (out_len + n > out_cap) {
-                    out_cap *= 2;
-                    char *tmp = realloc(out_buf, out_cap);
-                    if (!tmp) { free(out_buf); out_buf = NULL; break; }
-                    out_buf = tmp;
-                }
-                if (out_buf) {
-                    memcpy(out_buf + out_len, pull_buf, n);
-                    out_len += n;
+                if (cli_report_buffer_append(&out_buf, &out_len, &out_cap, pull_buf, n) != 0) {
+                    cli_disable_report_buffer(fout, &out_buf, &out_len, &out_cap);
+                    fwrite(pull_buf, 1, n, fout);
+                    try_report = 0;
                 }
             } else {
                 fwrite(pull_buf, 1, n, fout);
@@ -329,6 +1179,7 @@ int main(int argc, char **argv) {
         free(out_buf);
         if (fin != stdin) fclose(fin);
         if (fout != stdout) fclose(fout);
+        if (close_stats && fstats) fclose(fstats);
         tf_pipeline_free(p);
         return 1;
     }
@@ -338,15 +1189,10 @@ int main(int argc, char **argv) {
     size_t n;
     while ((n = tf_pipeline_pull(p, TF_CHAN_MAIN, pull_buf, sizeof(pull_buf))) > 0) {
         if (try_report && out_buf) {
-            while (out_len + n > out_cap) {
-                out_cap *= 2;
-                char *tmp = realloc(out_buf, out_cap);
-                if (!tmp) { free(out_buf); out_buf = NULL; break; }
-                out_buf = tmp;
-            }
-            if (out_buf) {
-                memcpy(out_buf + out_len, pull_buf, n);
-                out_len += n;
+            if (cli_report_buffer_append(&out_buf, &out_len, &out_cap, pull_buf, n) != 0) {
+                cli_disable_report_buffer(fout, &out_buf, &out_len, &out_cap);
+                fwrite(pull_buf, 1, n, fout);
+                try_report = 0;
             }
         } else {
             fwrite(pull_buf, 1, n, fout);
@@ -377,16 +1223,19 @@ int main(int argc, char **argv) {
         fwrite(pull_buf, 1, n, stderr);
     }
 
-    /* Pull stats to stderr (unless quiet) */
-    if (!quiet) {
+    /* Pull stats to the requested machine-readable path, or stderr unless quiet. */
+    FILE *stats_out = stats_json_file ? fstats : (!quiet ? stderr : NULL);
+    if (stats_out) {
         while ((n = tf_pipeline_pull(p, TF_CHAN_STATS, pull_buf, sizeof(pull_buf))) > 0) {
-            fwrite(pull_buf, 1, n, stderr);
+            fwrite(pull_buf, 1, n, stats_out);
         }
+        fflush(stats_out);
     }
 
     /* Cleanup */
     if (fin != stdin) fclose(fin);
     if (fout != stdout) fclose(fout);
+    if (close_stats && fstats) fclose(fstats);
     tf_pipeline_free(p);
     return 0;
 }