tranfi 0.1.2 → 0.2.1

package/csrc/ir_sql.c

@@ -21,34 +21,60 @@ typedef struct {
   char  *data;
   size_t len;
   size_t cap;
+  int    failed;
 } strbuf;
 
+static void sb_free(strbuf *sb);
+
 static void sb_init(strbuf *sb) {
-  sb->data = malloc(256);
+  sb->data = tf_mallocarray_checked(256, sizeof(char));
   sb->len = 0;
   sb->cap = 256;
+  sb->failed = 0;
   if (sb->data) sb->data[0] = '\0';
+  else { sb->cap = 0; sb->failed = 1; }
 }
 
-static void sb_ensure(strbuf *sb, size_t extra) {
-  if (sb->len + extra + 1 > sb->cap) {
-    size_t newcap = sb->cap * 2;
-    if (newcap < sb->len + extra + 1) newcap = sb->len + extra + 1;
-    char *nd = realloc(sb->data, newcap);
-    if (nd) { sb->data = nd; sb->cap = newcap; }
+static int sb_ensure(strbuf *sb, size_t extra) {
+  if (!sb || sb->failed) return TF_ERROR;
+  size_t need = 0;
+  if (tf_size_add(sb->len, extra, &need) != TF_OK ||
+      tf_size_add(need, 1, &need) != TF_OK) {
+    sb->failed = 1;
+    return TF_ERROR;
+  }
+  if (need <= sb->cap) return TF_OK;
+  size_t newcap = 0;
+  if (tf_size_grow_pow2(sb->cap, need, 256, &newcap) != TF_OK) {
+    sb->failed = 1;
+    return TF_ERROR;
   }
+  char *nd = tf_reallocarray_checked(sb->data, newcap, sizeof(char));
+  if (!nd) {
+    sb->failed = 1;
+    return TF_ERROR;
+  }
+  sb->data = nd;
+  sb->cap = newcap;
+  return TF_OK;
 }
 
 static void sb_append(strbuf *sb, const char *s) {
+  if (!sb || sb->failed || !s) return;
   size_t n = strlen(s);
-  sb_ensure(sb, n);
+  if (sb_ensure(sb, n) != TF_OK) return;
   memcpy(sb->data + sb->len, s, n);
   sb->len += n;
   sb->data[sb->len] = '\0';
 }
 
 static void sb_appendn(strbuf *sb, const char *s, size_t n) {
-  sb_ensure(sb, n);
+  if (!sb || sb->failed || (!s && n > 0)) {
+    if (sb) sb->failed = 1;
+    return;
+  }
+  if (sb_ensure(sb, n) != TF_OK) return;
+  if (n == 0) return;
   memcpy(sb->data + sb->len, s, n);
   sb->len += n;
   sb->data[sb->len] = '\0';
@@ -58,36 +84,103 @@ static void sb_appendf(strbuf *sb, const char *fmt, ...)
     __attribute__((format(printf, 2, 3)));
 
 static void sb_appendf(strbuf *sb, const char *fmt, ...) {
+  if (!sb || sb->failed || !fmt) return;
   char buf[512];
   va_list ap;
   va_start(ap, fmt);
   int n = vsnprintf(buf, sizeof(buf), fmt, ap);
   va_end(ap);
-  if (n > 0) sb_appendn(sb, buf, (size_t)n);
+  if (n == 0) return;
+  if (n < 0) {
+    sb->failed = 1;
+    return;
+  }
+  if ((size_t)n < sizeof(buf)) {
+    sb_appendn(sb, buf, (size_t)n);
+    return;
+  }
+  size_t dyn_size = 0;
+  if (tf_size_add((size_t)n, 1, &dyn_size) != TF_OK) {
+    sb->failed = 1;
+    return;
+  }
+  char *dyn = tf_mallocarray_checked(dyn_size, sizeof(char));
+  if (!dyn) {
+    sb->failed = 1;
+    return;
+  }
+  va_start(ap, fmt);
+  int n2 = vsnprintf(dyn, dyn_size, fmt, ap);
+  va_end(ap);
+  if (n2 < 0 || (size_t)n2 >= dyn_size) {
+    free(dyn);
+    sb->failed = 1;
+    return;
+  }
+  sb_appendn(sb, dyn, (size_t)n2);
+  free(dyn);
 }
 
 static char *sb_detach(strbuf *sb) {
+  if (!sb || sb->failed) {
+    if (sb) sb_free(sb);
+    return NULL;
+  }
   char *s = sb->data;
   sb->data = NULL;
   sb->len = sb->cap = 0;
+  sb->failed = 0;
   return s;
 }
 
 static void sb_free(strbuf *sb) {
+  if (!sb) return;
   free(sb->data);
   sb->data = NULL;
   sb->len = sb->cap = 0;
+  sb->failed = 0;
 }
 
 /* ---- Expression AST to SQL ---- */
 
-/* Append a SQL-quoted identifier: "name" */
-static void sql_quote_ident(strbuf *sb, const char *name) {
-  sb_append(sb, "\"");
-  for (const char *p = name; *p; p++) {
+static void sql_quote_ident_part(strbuf *sb, const char *name) {
+  const char *safe = name ? name : "";
+  for (const char *p = safe; *p; p++) {
     if (*p == '"') sb_append(sb, "\"\"");
     else sb_appendn(sb, p, 1);
   }
+}
+
+static void sql_quote_ident_part_range(strbuf *sb, const char *name, size_t len) {
+  if (!name && len > 0) {
+    if (sb) sb->failed = 1;
+    return;
+  }
+  for (size_t i = 0; i < len; i++) {
+    if (name[i] == '"') sb_append(sb, "\"\"");
+    else sb_appendn(sb, name + i, 1);
+  }
+}
+
+/* Append a SQL-quoted identifier: "name" */
+static void sql_quote_ident(strbuf *sb, const char *name) {
+  sb_append(sb, "\"");
+  sql_quote_ident_part(sb, name);
+  sb_append(sb, "\"");
+}
+
+static void sql_quote_ident_range(strbuf *sb, const char *name, size_t len) {
+  sb_append(sb, "\"");
+  sql_quote_ident_part_range(sb, name, len);
+  sb_append(sb, "\"");
+}
+
+static void sql_quote_joined_ident(strbuf *sb, const char *prefix,
+                                   const char *middle, const char *suffix) {
+  sb_append(sb, "\"");
+  sql_quote_ident_part(sb, prefix);
+  sql_quote_ident_part(sb, middle);
+  sql_quote_ident_part(sb, suffix);
   sb_append(sb, "\"");
 }
 
@@ -121,13 +214,17 @@ static int expr_to_sql(const tf_expr *e, strbuf *sb) {
       return 0;
 
     case EXPR_LIT_FLOAT:
-      sb_appendf(sb, "%g", e->lit_float);
+      sb_appendf(sb, TF_FLOAT64_ROUNDTRIP_FORMAT, e->lit_float);
       return 0;
 
     case EXPR_LIT_STR:
       sql_quote_str(sb, e->lit_str);
       return 0;
 
+    case EXPR_LIT_BOOL:
+      sb_append(sb, e->lit_bool ? "TRUE" : "FALSE");
+      return 0;
+
     case EXPR_COL_REF:
       sql_quote_ident(sb, e->col_name);
       return 0;
@@ -224,6 +321,103 @@ static int expr_to_sql(const tf_expr *e, strbuf *sb) {
         return 0;
       }
 
+      /* Special: case_when(cond, value, ..., default) → searched CASE */
+      if (strcmp(name, "case_when") == 0 && n >= 2) {
+        int pair_count = n / 2;
+        int default_idx = (n % 2 == 1) ? n - 1 : -1;
+        sb_append(sb, "(CASE");
+        for (int i = 0; i < pair_count; i++) {
+          sb_append(sb, " WHEN ");
+          if (expr_to_sql(e->func.args[i * 2], sb) != 0) return -1;
+          sb_append(sb, " THEN ");
+          if (expr_to_sql(e->func.args[i * 2 + 1], sb) != 0) return -1;
+        }
+        sb_append(sb, " ELSE ");
+        if (default_idx >= 0) {
+          if (expr_to_sql(e->func.args[default_idx], sb) != 0) return -1;
+        } else {
+          sb_append(sb, "NULL");
+        }
+        sb_append(sb, " END)");
+        return 0;
+      }
+
+      /* Special: case_match(value, key, result, ..., default) → simple CASE */
+      if (strcmp(name, "case_match") == 0 && n >= 3) {
+        int remaining = n - 1;
+        int pair_count = remaining / 2;
+        int default_idx = (remaining % 2 == 1) ? n - 1 : -1;
+        sb_append(sb, "(CASE ");
+        if (expr_to_sql(e->func.args[0], sb) != 0) return -1;
+        for (int i = 0; i < pair_count; i++) {
+          sb_append(sb, " WHEN ");
+          if (expr_to_sql(e->func.args[1 + i * 2], sb) != 0) return -1;
+          sb_append(sb, " THEN ");
+          if (expr_to_sql(e->func.args[2 + i * 2], sb) != 0) return -1;
+        }
+        sb_append(sb, " ELSE ");
+        if (default_idx >= 0) {
+          if (expr_to_sql(e->func.args[default_idx], sb) != 0) return -1;
+        } else {
+          sb_append(sb, "NULL");
+        }
+        sb_append(sb, " END)");
+        return 0;
+      }
+
+      /* Special: if_any(pred, ...) / if_all(pred, ...) → OR/AND reduction */
+      if (strcmp(name, "if_any") == 0 || strcmp(name, "if_all") == 0) {
+        int is_any = strcmp(name, "if_any") == 0;
+        if (n == 0) {
+          sb_append(sb, is_any ? "(FALSE)" : "(TRUE)");
+          return 0;
+        }
+        sb_append(sb, "(");
+        for (int i = 0; i < n; i++) {
+          if (i > 0) sb_append(sb, is_any ? " OR " : " AND ");
+          if (expr_to_sql(e->func.args[i], sb) != 0) return -1;
+        }
+        sb_append(sb, ")");
+        return 0;
+      }
+
+      /* Special: between(x, left, right) / inrange(x, left, right) → inclusive SQL range */
+      if ((strcmp(name, "between") == 0 || strcmp(name, "inrange") == 0) && n == 3) {
+        sb_append(sb, "(");
+        if (expr_to_sql(e->func.args[0], sb) != 0) return -1;
+        sb_append(sb, " BETWEEN ");
+        if (expr_to_sql(e->func.args[1], sb) != 0) return -1;
+        sb_append(sb, " AND ");
+        if (expr_to_sql(e->func.args[2], sb) != 0) return -1;
+        sb_append(sb, ")");
+        return 0;
+      }
+
+      /* Special: date/time component extraction */
+      if ((strcmp(name, "year") == 0 || strcmp(name, "month") == 0 ||
+           strcmp(name, "day") == 0 || strcmp(name, "hour") == 0 ||
+           strcmp(name, "minute") == 0 || strcmp(name, "second") == 0 ||
+           strcmp(name, "weekday") == 0 || strcmp(name, "epoch") == 0) && n == 1) {
+        const char *part = name;
+        if (strcmp(name, "weekday") == 0) part = "dow";
+        sb_append(sb, "EXTRACT(");
+        sb_append(sb, part);
+        sb_append(sb, " FROM ");
+        if (expr_to_sql(e->func.args[0], sb) != 0) return -1;
+        sb_append(sb, ")");
+        return 0;
+      }
+
+      /* Special: date_trunc(value, 'unit') → SQL date_trunc('unit', value) */
+      if (strcmp(name, "date_trunc") == 0 && n == 2 && e->func.args[1]->kind == EXPR_LIT_STR) {
+        sb_append(sb, "date_trunc(");
+        sql_quote_str(sb, e->func.args[1]->lit_str);
+        sb_append(sb, ", ");
+        if (expr_to_sql(e->func.args[0], sb) != 0) return -1;
+        sb_append(sb, ")");
+        return 0;
+      }
+
       /* Special: mod(a, b) → (a % b) */
       if (strcmp(name, "mod") == 0 && n == 2) {
         sb_append(sb, "(");
@@ -275,7 +469,7 @@ static char *translate_expr(const char *expr_str) {
   sb_init(&sb);
   int rc = expr_to_sql(e, &sb);
   tf_expr_free(e);
-  if (rc != 0) { sb_free(&sb); return NULL; }
+  if (rc != 0 || sb.failed) { sb_free(&sb); return NULL; }
   return sb_detach(&sb);
 }
 
@@ -300,6 +494,138 @@ static int jbool(const cJSON *obj, const char *key, int def) {
   return def;
 }
 
+static const char *sql_type_for_cast(const char *tf_type) {
+  if (!tf_type) return "VARCHAR";
+  if (strcmp(tf_type, "int") == 0 || strcmp(tf_type, "int64") == 0) return "BIGINT";
+  if (strcmp(tf_type, "float") == 0 || strcmp(tf_type, "float64") == 0) return "DOUBLE";
+  if (strcmp(tf_type, "bool") == 0 || strcmp(tf_type, "boolean") == 0) return "BOOLEAN";
+  if (strcmp(tf_type, "string") == 0 || strcmp(tf_type, "str") == 0) return "VARCHAR";
+  if (strcmp(tf_type, "date") == 0) return "DATE";
+  if (strcmp(tf_type, "timestamp") == 0 || strcmp(tf_type, "datetime") == 0) return "TIMESTAMP";
+  return "VARCHAR";
+}
+
+static const char *cast_on_error_policy(const cJSON *args) {
+  const cJSON *j = args ? cJSON_GetObjectItemCaseSensitive(args, "on_error") : NULL;
+  if (!j) j = args ? cJSON_GetObjectItemCaseSensitive(args, "onError") : NULL;
+  if (!j || !cJSON_IsString(j) || !j->valuestring) return "coerce";
+  if (strcmp(j->valuestring, "null") == 0 || strcmp(j->valuestring, "nulling") == 0) return "null";
+  if (strcmp(j->valuestring, "fail") == 0 || strcmp(j->valuestring, "error") == 0 ||
+      strcmp(j->valuestring, "strict") == 0) return "fail";
+  return "coerce";
+}
+
+static int append_cast_expr(strbuf *sb, const char *column,
+                            const char *tf_type, const char *policy) {
+  if (!sb || !column) return TF_ERROR;
+  const char *sql_type = sql_type_for_cast(tf_type);
+  strbuf qcol;
+  sb_init(&qcol);
+  sql_quote_ident(&qcol, column);
+  if (qcol.failed) {
+    sb_free(&qcol);
+    return TF_ERROR;
+  }
+
+  if (strcmp(policy, "null") == 0) {
+    sb_append(sb, "TRY_CAST(");
+    sb_append(sb, qcol.data);
+    sb_appendf(sb, " AS %s)", sql_type);
+    sb_free(&qcol);
+    return sb->failed ? TF_ERROR : TF_OK;
+  }
+
+  if (strcmp(policy, "fail") == 0) {
+    sb_append(sb, "CAST(");
+    sb_append(sb, qcol.data);
+    sb_appendf(sb, " AS %s)", sql_type);
+    sb_free(&qcol);
+    return sb->failed ? TF_ERROR : TF_OK;
+  }
+
+  if (strcmp(sql_type, "BIGINT") == 0) {
+    sb_append(sb, "COALESCE(TRY_CAST(regexp_extract(CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS VARCHAR), ");
+    sql_quote_str(sb, "^[[:space:]]*[+-]?[0-9]+");
+    sb_append(sb, ") AS BIGINT), 0)");
+  } else if (strcmp(sql_type, "DOUBLE") == 0) {
+    sb_append(sb, "COALESCE(TRY_CAST(regexp_extract(CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS VARCHAR), ");
+    sql_quote_str(sb, "^[[:space:]]*[+-]?([0-9]+(\\.[0-9]*)?|\\.[0-9]+)([eE][+-]?[0-9]+)?");
+    sb_append(sb, ") AS DOUBLE), 0.0)");
+  } else if (strcmp(sql_type, "BOOLEAN") == 0) {
+    sb_append(sb, "(CASE WHEN ");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " IS NULL THEN NULL WHEN CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS VARCHAR) = '' OR CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS VARCHAR) = 'false' THEN FALSE ELSE TRUE END)");
+  } else if (strcmp(sql_type, "DATE") == 0) {
+    sb_append(sb, "COALESCE(TRY_CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS DATE), DATE '1970-01-01')");
+  } else if (strcmp(sql_type, "TIMESTAMP") == 0) {
+    sb_append(sb, "COALESCE(TRY_CAST(");
+    sb_append(sb, qcol.data);
+    sb_append(sb, " AS TIMESTAMP), TIMESTAMP '1970-01-01 00:00:00')");
+  } else {
+    sb_append(sb, "CAST(");
+    sb_append(sb, qcol.data);
+    sb_appendf(sb, " AS %s)", sql_type);
+  }
+  sb_free(&qcol);
+  return sb->failed ? TF_ERROR : TF_OK;
+}
+
+static int append_frequency_key_expr(strbuf *sb, cJSON *cols) {
+  if (!sb || !cJSON_IsArray(cols) || cJSON_GetArraySize(cols) <= 0) return TF_ERROR;
+  int n = cJSON_GetArraySize(cols);
+  for (int i = 0; i < n; i++) {
+    cJSON *c = cJSON_GetArrayItem(cols, i);
+    if (!cJSON_IsString(c) || !c->valuestring || c->valuestring[0] == '\0') {
+      return TF_ERROR;
+    }
+    if (i > 0) sb_append(sb, " || chr(1) || ");
+    sb_append(sb, "COALESCE(CAST(");
+    sql_quote_ident(sb, c->valuestring);
+    sb_append(sb, " AS VARCHAR), ");
+    sql_quote_str(sb, "\\N");
+    sb_append(sb, ")");
+    if (sb->failed) return TF_ERROR;
+  }
+  return TF_OK;
+}
+
+static int append_grep_match_expr(strbuf *sb, const char *column,
+                                  const char *pattern, int regex) {
+  if (!sb || !column || !pattern) return TF_ERROR;
+  strbuf qcol;
+  sb_init(&qcol);
+  sql_quote_ident(&qcol, column);
+  if (qcol.failed) {
+    sb_free(&qcol);
+    return TF_ERROR;
+  }
+
+  sb_append(sb, "(CASE WHEN typeof(");
+  sb_append(sb, qcol.data);
+  sb_append(sb, ") = 'VARCHAR' THEN COALESCE(");
+  if (regex) {
+    sb_append(sb, "regexp_matches(CAST(");
+  } else {
+    sb_append(sb, "contains(CAST(");
+  }
+  sb_append(sb, qcol.data);
+  sb_append(sb, " AS VARCHAR), ");
+  sql_quote_str(sb, pattern);
+  sb_append(sb, "), FALSE) ELSE FALSE END)");
+  sb_free(&qcol);
+  return sb->failed ? TF_ERROR : TF_OK;
+}
+
 /* Emit a CTE for a transform op. prev is the name of the previous CTE/source.
  * Appends SQL like: step_N AS (SELECT ... FROM prev ...) */
 static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
@@ -316,10 +642,48 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     return 0;
   }
 
+  /* ---- relocate ---- */
+  if (strcmp(op, "relocate") == 0) {
+    cJSON *before = cJSON_GetObjectItemCaseSensitive(args, "before");
+    cJSON *after = cJSON_GetObjectItemCaseSensitive(args, "after");
+    if (before || after) {
+      *error = strdup("relocate: SQL lowering for before/after requires known schema; default-front relocate is supported");
+      return -1;
+    }
+    cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
+    if (!cols || !cJSON_IsArray(cols)) { *error = strdup("relocate: missing 'columns'"); return -1; }
+    if (tf_column_selectors_have_syntax_json(cols)) {
+      *error = strdup("relocate: selector helpers require known schema for SQL lowering");
+      return -1;
+    }
+    strbuf moved;
+    sb_init(&moved);
+    int n = cJSON_GetArraySize(cols);
+    for (int i = 0; i < n; i++) {
+      cJSON *c = cJSON_GetArrayItem(cols, i);
+      if (!cJSON_IsString(c)) continue;
+      if (moved.len > 0) sb_append(&moved, ", ");
+      sql_quote_ident(&moved, c->valuestring);
+    }
+    if (moved.len == 0) {
+      sb_free(&moved);
+      *error = strdup("relocate: missing 'columns'");
+      return -1;
+    }
+    sb_appendf(sb, "%s AS (SELECT %s, * EXCLUDE (%s) FROM %s)",
+               cte_name, moved.data, moved.data, prev);
+    sb_free(&moved);
+    return 0;
+  }
+
   /* ---- select / reorder ---- */
   if (strcmp(op, "select") == 0 || strcmp(op, "reorder") == 0) {
     cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
     if (!cols || !cJSON_IsArray(cols)) { *error = strdup("select: missing 'columns'"); return -1; }
+    if (tf_column_selectors_have_syntax_json(cols)) {
+      *error = strdup("select: selector helpers require known schema for SQL lowering");
+      return -1;
+    }
     strbuf sel;
     sb_init(&sel);
     int n = cJSON_GetArraySize(cols);
@@ -393,6 +757,18 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
 
   /* ---- unique / dedup ---- */
   if (strcmp(op, "unique") == 0 || strcmp(op, "dedup") == 0) {
+    cJSON *sorted = cJSON_GetObjectItemCaseSensitive(args, "sorted");
+    if (cJSON_IsBool(sorted) && cJSON_IsTrue(sorted)) {
+      *error = strdup("unique: sorted=true adjacent-run mode cannot be lowered to SQL");
+      return -1;
+    }
+    cJSON *mode = cJSON_GetObjectItemCaseSensitive(args, "mode");
+    cJSON *approx = cJSON_GetObjectItemCaseSensitive(args, "approx");
+    if ((cJSON_IsString(mode) && mode->valuestring && strcmp(mode->valuestring, "approx") == 0) ||
+        (cJSON_IsBool(approx) && cJSON_IsTrue(approx))) {
+      *error = strdup("unique: approximate mode cannot be lowered to SQL");
+      return -1;
+    }
     cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
     if (cols && cJSON_IsArray(cols) && cJSON_GetArraySize(cols) > 0) {
       strbuf dcols;
@@ -434,7 +810,7 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
   }
 
   /* ---- head ---- */
-  if (strcmp(op, "head") == 0) {
+  if (strcmp(op, "head") == 0 || strcmp(op, "slice-head") == 0) {
     int n = jint(args, "n", 10);
     sb_appendf(sb, "%s AS (SELECT * FROM %s LIMIT %d)", cte_name, prev, n);
     return 0;
@@ -448,7 +824,7 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
   }
 
   /* ---- tail ---- */
-  if (strcmp(op, "tail") == 0) {
+  if (strcmp(op, "tail") == 0 || strcmp(op, "slice-tail") == 0) {
     int n = jint(args, "n", 10);
     sb_appendf(sb, "%s AS (SELECT * FROM (SELECT *, ROW_NUMBER() OVER () AS _rn, "
                "COUNT(*) OVER () AS _total FROM %s) WHERE _rn > _total - %d)",
@@ -456,12 +832,14 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     return 0;
   }
 
-  /* ---- top ---- */
-  if (strcmp(op, "top") == 0) {
+  /* ---- bounded top-k / bottom-k ---- */
+  if (strcmp(op, "top") == 0 || strcmp(op, "top-k") == 0 ||
+      strcmp(op, "bottom-k") == 0 || strcmp(op, "slice-min") == 0 ||
+      strcmp(op, "slice-max") == 0) {
     int n = jint(args, "n", 10);
     const char *column = jstr(args, "column");
     int desc = jbool(args, "desc", 1);
-    if (!column) { *error = strdup("top: missing 'column'"); return -1; }
+    if (!column) { *error = strdup("top-k: missing 'column'"); return -1; }
     strbuf tmp;
     sb_init(&tmp);
     sql_quote_ident(&tmp, column);
@@ -473,9 +851,8 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
 
   /* ---- sample ---- */
   if (strcmp(op, "sample") == 0) {
-    int n = jint(args, "n", 100);
-    sb_appendf(sb, "%s AS (SELECT * FROM %s USING SAMPLE %d)", cte_name, prev, n);
-    return 0;
+    *error = strdup("sample: deterministic reservoir sampling cannot be lowered to SQL");
+    return -1;
   }
 
   /* ---- grep ---- */
@@ -486,28 +863,16 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     int regex = jbool(args, "regex", 0);
     if (!pattern) { *error = strdup("grep: missing 'pattern'"); return -1; }
     if (!column) column = "_line";
-    strbuf tmp;
-    sb_init(&tmp);
-    sql_quote_ident(&tmp, column);
-    if (regex) {
-      sb_appendf(sb, "%s AS (SELECT * FROM %s WHERE %sregexp_matches(%s, ",
-                 cte_name, prev, invert ? "NOT " : "", tmp.data);
-      sql_quote_str(sb, pattern);
-      sb_append(sb, "))");
-    } else {
-      sb_appendf(sb, "%s AS (SELECT * FROM %s WHERE ", cte_name, prev);
-      if (invert) sb_append(sb, "NOT ");
-      sb_append(sb, tmp.data);
-      sb_append(sb, " LIKE '%");
-      /* Escape pattern for LIKE */
-      for (const char *p = pattern; *p; p++) {
-        if (*p == '%' || *p == '_' || *p == '\\') sb_appendn(sb, "\\", 1);
-        if (*p == '\'') sb_append(sb, "''");
-        else sb_appendn(sb, p, 1);
-      }
-      sb_append(sb, "%')");
+    strbuf pred;
+    sb_init(&pred);
+    if (append_grep_match_expr(&pred, column, pattern, regex) != TF_OK) {
+      sb_free(&pred);
+      *error = strdup("sql: out of memory");
+      return -1;
     }
-    sb_free(&tmp);
+    sb_appendf(sb, "%s AS (SELECT * FROM %s WHERE %s%s)",
+               cte_name, prev, invert ? "NOT " : "", pred.data);
+    sb_free(&pred);
     return 0;
   }
 
@@ -515,48 +880,23 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
   if (strcmp(op, "cast") == 0) {
     cJSON *mapping = cJSON_GetObjectItemCaseSensitive(args, "mapping");
     if (!mapping) { *error = strdup("cast: missing 'mapping'"); return -1; }
-    strbuf cols;
-    sb_init(&cols);
-    cJSON *item = NULL;
-    cJSON_ArrayForEach(item, mapping) {
-      if (!cJSON_IsString(item)) continue;
-      sb_append(&cols, ", CAST(");
-      sql_quote_ident(&cols, item->string);
-      /* Map tranfi types to SQL types */
-      const char *tf_type = item->valuestring;
-      const char *sql_type = "VARCHAR";
-      if (strcmp(tf_type, "int") == 0 || strcmp(tf_type, "int64") == 0) sql_type = "BIGINT";
-      else if (strcmp(tf_type, "float") == 0 || strcmp(tf_type, "float64") == 0) sql_type = "DOUBLE";
-      else if (strcmp(tf_type, "bool") == 0) sql_type = "BOOLEAN";
-      else if (strcmp(tf_type, "string") == 0) sql_type = "VARCHAR";
-      else if (strcmp(tf_type, "date") == 0) sql_type = "DATE";
-      else if (strcmp(tf_type, "timestamp") == 0) sql_type = "TIMESTAMP";
-      sb_appendf(&cols, " AS %s) AS ", sql_type);
-      sql_quote_ident(&cols, item->string);
-    }
-    /* Use COLUMNS(*) EXCLUDE + explicit casts — simpler: use REPLACE */
-    /* DuckDB REPLACE: SELECT * REPLACE (CAST(col AS type) AS col) */
+    const char *policy = cast_on_error_policy(args);
     strbuf rep;
     sb_init(&rep);
     int first = 1;
+    cJSON *item = NULL;
     cJSON_ArrayForEach(item, mapping) {
       if (!cJSON_IsString(item)) continue;
       if (!first) sb_append(&rep, ", ");
       first = 0;
-      const char *tf_type = item->valuestring;
-      const char *sql_type = "VARCHAR";
-      if (strcmp(tf_type, "int") == 0 || strcmp(tf_type, "int64") == 0) sql_type = "BIGINT";
-      else if (strcmp(tf_type, "float") == 0 || strcmp(tf_type, "float64") == 0) sql_type = "DOUBLE";
-      else if (strcmp(tf_type, "bool") == 0) sql_type = "BOOLEAN";
-      else if (strcmp(tf_type, "string") == 0) sql_type = "VARCHAR";
-      else if (strcmp(tf_type, "date") == 0) sql_type = "DATE";
-      else if (strcmp(tf_type, "timestamp") == 0) sql_type = "TIMESTAMP";
-      sb_append(&rep, "CAST(");
-      sql_quote_ident(&rep, item->string);
-      sb_appendf(&rep, " AS %s) AS ", sql_type);
+      if (append_cast_expr(&rep, item->string, item->valuestring, policy) != TF_OK) {
+        sb_free(&rep);
+        *error = strdup("sql: out of memory");
+        return -1;
+      }
+      sb_append(&rep, " AS ");
       sql_quote_ident(&rep, item->string);
     }
-    sb_free(&cols);
     sb_appendf(sb, "%s AS (SELECT * REPLACE (%s) FROM %s)", cte_name, rep.data, prev);
     sb_free(&rep);
     return 0;
@@ -570,28 +910,26 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     cJSON *max_v = cJSON_GetObjectItemCaseSensitive(args, "max");
     strbuf expr;
     sb_init(&expr);
-    sql_quote_ident(&expr, column);
     if (min_v && max_v) {
       strbuf tmp;
       sb_init(&tmp);
       sql_quote_ident(&tmp, column);
-      sb_init(&expr);
-      sb_appendf(&expr, "GREATEST(%g, LEAST(%g, %s))", min_v->valuedouble, max_v->valuedouble, tmp.data);
+      sb_appendf(&expr, "GREATEST(" TF_FLOAT64_ROUNDTRIP_FORMAT ", LEAST(" TF_FLOAT64_ROUNDTRIP_FORMAT ", %s))", min_v->valuedouble, max_v->valuedouble, tmp.data);
       sb_free(&tmp);
     } else if (min_v) {
       strbuf tmp;
       sb_init(&tmp);
       sql_quote_ident(&tmp, column);
-      sb_init(&expr);
-      sb_appendf(&expr, "GREATEST(%g, %s)", min_v->valuedouble, tmp.data);
+      sb_appendf(&expr, "GREATEST(" TF_FLOAT64_ROUNDTRIP_FORMAT ", %s)", min_v->valuedouble, tmp.data);
       sb_free(&tmp);
     } else if (max_v) {
       strbuf tmp;
       sb_init(&tmp);
       sql_quote_ident(&tmp, column);
-      sb_init(&expr);
-      sb_appendf(&expr, "LEAST(%g, %s)", max_v->valuedouble, tmp.data);
+      sb_appendf(&expr, "LEAST(" TF_FLOAT64_ROUNDTRIP_FORMAT ", %s)", max_v->valuedouble, tmp.data);
       sb_free(&tmp);
+    } else {
+      sql_quote_ident(&expr, column);
     }
     strbuf qcol;
     sb_init(&qcol);
@@ -640,27 +978,33 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
   /* ---- trim ---- */
   if (strcmp(op, "trim") == 0) {
     cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
-    if (cols && cJSON_IsArray(cols) && cJSON_GetArraySize(cols) > 0) {
-      strbuf rep;
-      sb_init(&rep);
-      int n = cJSON_GetArraySize(cols);
-      for (int i = 0; i < n; i++) {
-        if (i > 0) sb_append(&rep, ", ");
-        cJSON *c = cJSON_GetArrayItem(cols, i);
-        if (!cJSON_IsString(c)) continue;
-        sb_append(&rep, "trim(");
-        sql_quote_ident(&rep, c->valuestring);
-        sb_append(&rep, ") AS ");
-        sql_quote_ident(&rep, c->valuestring);
+    if (!cols || !cJSON_IsArray(cols) || cJSON_GetArraySize(cols) <= 0) {
+      *error = strdup("trim: SQL lowering requires explicit columns");
+      return -1;
+    }
+    strbuf rep;
+    sb_init(&rep);
+    int n = cJSON_GetArraySize(cols);
+    for (int i = 0; i < n; i++) {
+      cJSON *c = cJSON_GetArrayItem(cols, i);
+      if (!cJSON_IsString(c) || !c->valuestring || c->valuestring[0] == '\0') {
+        sb_free(&rep);
+        *error = strdup("trim: invalid columns");
+        return -1;
       }
-      sb_appendf(sb, "%s AS (SELECT * REPLACE (%s) FROM %s)", cte_name, rep.data, prev);
+      if (i > 0) sb_append(&rep, ", ");
+      sb_append(&rep, "trim(");
+      sql_quote_ident(&rep, c->valuestring);
+      sb_append(&rep, ") AS ");
+      sql_quote_ident(&rep, c->valuestring);
+    }
+    if (rep.failed) {
       sb_free(&rep);
-    } else {
-      /* Trim all string columns — without schema we just pass through.
-       * DuckDB doesn't have a "trim all" so we'd need column names.
-       * Fallback: SELECT * (no trim) with a comment. */
-      sb_appendf(sb, "%s AS (SELECT * FROM %s)", cte_name, prev);
+      *error = strdup("sql: out of memory");
+      return -1;
     }
+    sb_appendf(sb, "%s AS (SELECT * REPLACE (%s) FROM %s)", cte_name, rep.data, prev);
+    sb_free(&rep);
     return 0;
   }
 
@@ -724,13 +1068,12 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
       else if (strcmp(func, "median") == 0) sql_func = "MEDIAN";
       sb_append(&sel, sql_func);
       sb_append(&sel, "(");
-      sql_quote_ident(&sel, col);
+      if (strcmp(func, "count") == 0 && strcmp(col, "*") == 0) sb_append(&sel, "*");
+      else sql_quote_ident(&sel, col);
       sb_append(&sel, ") AS ");
       if (result) sql_quote_ident(&sel, result);
       else {
-        char buf[256];
-        snprintf(buf, sizeof(buf), "%s_%s", func, col);
-        sql_quote_ident(&sel, buf);
+        sql_quote_joined_ident(&sel, col, "_", func);
       }
     }
     /* GROUP BY clause */
@@ -749,40 +1092,53 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
 
   /* ---- frequency ---- */
   if (strcmp(op, "frequency") == 0) {
+    cJSON *mode = args ? cJSON_GetObjectItemCaseSensitive(args, "mode") : NULL;
+    if (cJSON_IsString(mode) && strcmp(mode->valuestring, "approx") == 0) {
+      *error = strdup("frequency: mode=approx is not supported by SQL lowering");
+      return -1;
+    }
+    cJSON *approx = args ? cJSON_GetObjectItemCaseSensitive(args, "approx") : NULL;
+    if (cJSON_IsTrue(approx)) {
+      *error = strdup("frequency: mode=approx is not supported by SQL lowering");
+      return -1;
+    }
+    cJSON *overflow = args ? cJSON_GetObjectItemCaseSensitive(args, "overflow") : NULL;
+    if (cJSON_IsString(overflow) && strcmp(overflow->valuestring, "other") == 0) {
+      *error = strdup("frequency: overflow=other is not supported by SQL lowering");
+      return -1;
+    }
+
     cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
-    if (cols && cJSON_IsArray(cols) && cJSON_GetArraySize(cols) > 0) {
-      strbuf sel;
-      sb_init(&sel);
-      strbuf grp;
-      sb_init(&grp);
-      int n = cJSON_GetArraySize(cols);
-      for (int i = 0; i < n; i++) {
-        if (i > 0) { sb_append(&sel, ", "); sb_append(&grp, ", "); }
-        cJSON *c = cJSON_GetArrayItem(cols, i);
-        if (cJSON_IsString(c)) {
-          sql_quote_ident(&sel, c->valuestring);
-          sql_quote_ident(&grp, c->valuestring);
-        }
-      }
-      sb_appendf(sb, "%s AS (SELECT %s, COUNT(*) AS \"count\" FROM %s GROUP BY %s ORDER BY \"count\" DESC)",
-                 cte_name, sel.data, prev, grp.data);
-      sb_free(&sel);
-      sb_free(&grp);
-    } else {
-      /* No columns specified — frequency of all columns not meaningful,
-       * fall back to counting rows */
-      sb_appendf(sb, "%s AS (SELECT COUNT(*) AS \"count\" FROM %s)", cte_name, prev);
+    if (!cols || !cJSON_IsArray(cols) || cJSON_GetArraySize(cols) <= 0) {
+      *error = strdup("frequency: SQL lowering requires explicit columns");
+      return -1;
     }
+    strbuf key;
+    sb_init(&key);
+    if (append_frequency_key_expr(&key, cols) != TF_OK) {
+      sb_free(&key);
+      *error = strdup("frequency: invalid columns");
+      return -1;
+    }
+    sb_appendf(sb, "%s AS (SELECT \"value\", COUNT(*) AS \"count\" FROM "
+                 "(SELECT %s AS \"value\" FROM %s) __tf_frequency "
+                 "GROUP BY \"value\" ORDER BY \"count\" DESC, \"value\" ASC)",
+               cte_name, key.data, prev);
+    sb_free(&key);
     return 0;
   }
 
   /* ---- join ---- */
-  if (strcmp(op, "join") == 0) {
+  if (strcmp(op, "join") == 0 || strcmp(op, "semi-join") == 0 || strcmp(op, "anti-join") == 0) {
     const char *file = jstr(args, "file");
     const char *on = jstr(args, "on");
     const char *how = jstr(args, "how");
     if (!file || !on) { *error = strdup("join: missing 'file' or 'on'"); return -1; }
-    if (!how) how = "inner";
+    if (!how) {
+      if (strcmp(op, "semi-join") == 0) how = "semi";
+      else if (strcmp(op, "anti-join") == 0) how = "anti";
+      else how = "inner";
+    }
     const char *join_type = "INNER";
     if (strcmp(how, "left") == 0) join_type = "LEFT";
     else if (strcmp(how, "right") == 0) join_type = "RIGHT";
@@ -793,27 +1149,34 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     strbuf cond;
     sb_init(&cond);
     if (eq) {
-      char left_col[128], right_col[128];
+      const char *left = on;
       size_t llen = (size_t)(eq - on);
-      if (llen >= sizeof(left_col)) llen = sizeof(left_col) - 1;
-      memcpy(left_col, on, llen);
-      left_col[llen] = '\0';
-      strncpy(right_col, eq + 1, sizeof(right_col) - 1);
-      right_col[sizeof(right_col) - 1] = '\0';
-      /* Trim whitespace */
-      while (llen > 0 && left_col[llen - 1] == ' ') left_col[--llen] = '\0';
-      char *rp = right_col;
-      while (*rp == ' ') rp++;
+      while (llen > 0 && *left == ' ') { left++; llen--; }
+      while (llen > 0 && left[llen - 1] == ' ') llen--;
+
+      const char *right = eq + 1;
+      while (*right == ' ') right++;
+      size_t rlen = strlen(right);
+      while (rlen > 0 && right[rlen - 1] == ' ') rlen--;
       sb_append(&cond, "a.");
-      sql_quote_ident(&cond, left_col);
+      sql_quote_ident_range(&cond, left, llen);
       sb_append(&cond, " = b.");
-      sql_quote_ident(&cond, rp);
+      sql_quote_ident_range(&cond, right, rlen);
     } else {
       sb_append(&cond, "a.");
       sql_quote_ident(&cond, on);
       sb_append(&cond, " = b.");
       sql_quote_ident(&cond, on);
     }
+    if (strcmp(how, "semi") == 0 || strcmp(how, "anti") == 0) {
+      sb_appendf(sb, "%s AS (SELECT a.* FROM %s a WHERE %sEXISTS (SELECT 1 FROM read_csv_auto(",
+                 cte_name, prev, strcmp(how, "anti") == 0 ? "NOT " : "");
+      sql_quote_str(sb, file);
+      sb_appendf(sb, ") b WHERE %s))", cond.data);
+      sb_free(&cond);
+      return 0;
+    }
+
     sb_appendf(sb, "%s AS (SELECT a.* FROM %s a %s JOIN read_csv_auto(",
                cte_name, prev, join_type);
     sql_quote_str(sb, file);
@@ -822,6 +1185,29 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     return 0;
   }
 
+  /* ---- set ops ---- */
+  if (strcmp(op, "intersect") == 0 || strcmp(op, "setdiff") == 0 ||
+      strcmp(op, "intersect-all") == 0 || strcmp(op, "setdiff-all") == 0 ||
+      strcmp(op, "union") == 0 || strcmp(op, "union-all") == 0) {
+    const char *file = jstr(args, "file");
+    if (!file) { *error = strdup("set op: missing 'file'"); return -1; }
+    cJSON *cols = args ? cJSON_GetObjectItemCaseSensitive(args, "columns") : NULL;
+    if (cols && cJSON_IsArray(cols) && cJSON_GetArraySize(cols) > 0) {
+      *error = strdup("set op SQL lowering only supports all-column set operations");
+      return -1;
+    }
+    const char *sql_op = "INTERSECT";
+    if (strcmp(op, "setdiff") == 0) sql_op = "EXCEPT";
+    else if (strcmp(op, "intersect-all") == 0) sql_op = "INTERSECT ALL";
+    else if (strcmp(op, "setdiff-all") == 0) sql_op = "EXCEPT ALL";
+    else if (strcmp(op, "union") == 0) sql_op = "UNION";
+    else if (strcmp(op, "union-all") == 0) sql_op = "UNION ALL";
+    sb_appendf(sb, "%s AS (SELECT * FROM %s %s SELECT * FROM read_csv_auto(", cte_name, prev, sql_op);
+    sql_quote_str(sb, file);
+    sb_append(sb, "))");
+    return 0;
+  }
+
   /* ---- stack ---- */
   if (strcmp(op, "stack") == 0) {
     const char *file = jstr(args, "file");
@@ -841,9 +1227,9 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     strbuf qcol;
     sb_init(&qcol);
     sql_quote_ident(&qcol, column);
-    sb_appendf(sb, "%s AS (SELECT * REPLACE (unnest(string_split(%s, ", cte_name, qcol.data);
+    sb_appendf(sb, "%s AS (SELECT * REPLACE (unnest(CASE WHEN %s IS NULL THEN list_value(NULL::VARCHAR) ELSE string_split(%s, ", cte_name, qcol.data, qcol.data);
     sql_quote_str(sb, delimiter);
-    sb_appendf(sb, ")) AS %s) FROM %s)", qcol.data, prev);
+    sb_appendf(sb, ") END) AS %s) FROM %s)", qcol.data, prev);
     sb_free(&qcol);
     return 0;
   }
@@ -898,6 +1284,7 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     const char *name_col = jstr(args, "name_column");
     const char *val_col = jstr(args, "value_column");
     const char *agg = jstr(args, "agg");
+    cJSON *categories = args ? cJSON_GetObjectItemCaseSensitive(args, "categories") : NULL;
     if (!name_col || !val_col) { *error = strdup("pivot: missing args"); return -1; }
     if (!agg) agg = "first";
     const char *sql_agg = "FIRST";
@@ -910,8 +1297,55 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     sb_init(&qn); sb_init(&qv);
     sql_quote_ident(&qn, name_col);
     sql_quote_ident(&qv, val_col);
-    sb_appendf(sb, "%s AS (PIVOT %s ON %s USING %s(%s))",
-               cte_name, prev, qn.data, sql_agg, qv.data);
+    if (qn.failed || qv.failed) {
+      sb_free(&qn); sb_free(&qv);
+      *error = strdup("sql: out of memory");
+      return -1;
+    }
+    if (categories) {
+      if (!cJSON_IsArray(categories) || cJSON_GetArraySize(categories) <= 0) {
+        sb_free(&qn); sb_free(&qv);
+        *error = strdup("pivot: SQL categories must be a non-empty array");
+        return -1;
+      }
+      strbuf cats;
+      sb_init(&cats);
+      int n = cJSON_GetArraySize(categories);
+      for (int i = 0; i < n; i++) {
+        cJSON *cat = cJSON_GetArrayItem(categories, i);
+        if (!cJSON_IsString(cat)) {
+          sb_free(&cats); sb_free(&qn); sb_free(&qv);
+          *error = strdup("pivot: SQL categories must be strings");
+          return -1;
+        }
+        if (i > 0) sb_append(&cats, ", ");
+        sql_quote_str(&cats, cat->valuestring);
+        if (cats.failed) {
+          sb_free(&cats); sb_free(&qn); sb_free(&qv);
+          *error = strdup("sql: out of memory");
+          return -1;
+        }
+      }
+      sb_appendf(sb, "%s AS (PIVOT %s ON %s IN (%s) USING %s(%s))",
+                 cte_name, prev, qn.data, cats.data, sql_agg, qv.data);
+      sb_free(&cats);
+    } else {
+      if (strcmp(prev, "input_data") != 0) {
+        sb_free(&qn); sb_free(&qv);
+        *error = strdup("pivot: dynamic SQL pivot after upstream transforms requires categories=...");
+        return -1;
+      }
+      sb_appendf(sb,
+                 "%s AS (PIVOT %s ON %s IN (SELECT %s FROM "
+                 "(SELECT %s, row_number() OVER () AS \"__tf_pivot_category_ordinal\" FROM %s) __tf_pivot_cats "
+                 "GROUP BY %s ORDER BY min(\"__tf_pivot_category_ordinal\")) USING %s(%s))",
+                 cte_name, prev, qn.data, qn.data, qn.data, prev, qn.data, sql_agg, qv.data);
+    }
+    if (qn.failed || qv.failed || sb->failed) {
+      sb_free(&qn); sb_free(&qv);
+      *error = strdup("sql: out of memory");
+      return -1;
+    }
     sb_free(&qn); sb_free(&qv);
     return 0;
   }
@@ -932,18 +1366,18 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
       cJSON *b = cJSON_GetArrayItem(boundaries, i);
       double val = b->valuedouble;
       if (i == 0) {
-        sb_appendf(&expr, " WHEN %s < %g THEN '<%g'", qcol.data, val, val);
+        sb_appendf(&expr, " WHEN %s < " TF_FLOAT64_ROUNDTRIP_FORMAT " THEN '<" TF_FLOAT64_ROUNDTRIP_FORMAT "'", qcol.data, val, val);
       }
       if (i > 0) {
         cJSON *prev_b = cJSON_GetArrayItem(boundaries, i - 1);
-        sb_appendf(&expr, " WHEN %s >= %g AND %s < %g THEN '%g-%g'",
+        sb_appendf(&expr, " WHEN %s >= " TF_FLOAT64_ROUNDTRIP_FORMAT " AND %s < " TF_FLOAT64_ROUNDTRIP_FORMAT " THEN '" TF_FLOAT64_ROUNDTRIP_FORMAT "-" TF_FLOAT64_ROUNDTRIP_FORMAT "'",
                    qcol.data, prev_b->valuedouble, qcol.data, val,
                    prev_b->valuedouble, val);
       }
     }
     if (n > 0) {
       cJSON *last = cJSON_GetArrayItem(boundaries, n - 1);
-      sb_appendf(&expr, " WHEN %s >= %g THEN '>=%g'", qcol.data, last->valuedouble, last->valuedouble);
+      sb_appendf(&expr, " WHEN %s >= " TF_FLOAT64_ROUNDTRIP_FORMAT " THEN '" TF_FLOAT64_ROUNDTRIP_FORMAT "+'", qcol.data, last->valuedouble, last->valuedouble);
     }
     sb_append(&expr, " END");
     strbuf bin_col;
@@ -993,7 +1427,7 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
         strbuf qc;
         sb_init(&qc);
         sql_quote_ident(&qc, c->valuestring);
-        sb_appendf(&rep, "LAST_VALUE(%s IGNORE NULLS) OVER (ORDER BY rowid() ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS %s",
+        sb_appendf(&rep, "LAST_VALUE(%s IGNORE NULLS) OVER (ORDER BY _rn ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS %s",
                    qc.data, qc.data);
         sb_free(&qc);
       }
@@ -1005,25 +1439,70 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     return 0;
   }
 
-  /* ---- window ---- */
-  if (strcmp(op, "window") == 0) {
+  /* ---- window / rolling ---- */
+  if (strcmp(op, "window") == 0 || strcmp(op, "rolling-sum") == 0 ||
+      strcmp(op, "rolling-mean") == 0 || strcmp(op, "rolling-min") == 0 ||
+      strcmp(op, "rolling-max") == 0 || strcmp(op, "rolling-any") == 0 ||
+      strcmp(op, "rolling-all") == 0) {
     const char *column = jstr(args, "column");
     int size = jint(args, "size", 3);
     const char *func = jstr(args, "func");
     const char *result = jstr(args, "result");
+    int bool_roll = 0;
+    if (strcmp(op, "rolling-sum") == 0) func = "sum";
+    else if (strcmp(op, "rolling-mean") == 0) func = "avg";
+    else if (strcmp(op, "rolling-min") == 0) func = "min";
+    else if (strcmp(op, "rolling-max") == 0) func = "max";
+    else if (strcmp(op, "rolling-any") == 0) { func = "any"; bool_roll = 1; }
+    else if (strcmp(op, "rolling-all") == 0) { func = "all"; bool_roll = 1; }
     if (!column || !func) { *error = strdup("window: missing args"); return -1; }
-    const char *sql_func = "AVG";
-    if (strcmp(func, "sum") == 0) sql_func = "SUM";
-    else if (strcmp(func, "min") == 0) sql_func = "MIN";
-    else if (strcmp(func, "max") == 0) sql_func = "MAX";
-    else if (strcmp(func, "avg") == 0) sql_func = "AVG";
+
     strbuf qcol, qres;
     sb_init(&qcol); sb_init(&qres);
     sql_quote_ident(&qcol, column);
     if (result) sql_quote_ident(&qres, result);
-    else sb_appendf(&qres, "\"%s_%s_%d\"", func, column, size);
-    sb_appendf(sb, "%s AS (SELECT *, %s(%s) OVER (ORDER BY rowid() ROWS BETWEEN %d PRECEDING AND CURRENT ROW) AS %s FROM %s)",
-               cte_name, sql_func, qcol.data, size - 1, qres.data, prev);
+    else sb_appendf(&qres, "\"%s_%s%d\"", column, func, size);
+
+    int preceding = size > 0 ? size - 1 : 0;
+    if (bool_roll) {
+      const char *nulls = jstr(args, "nulls");
+      if (!nulls) nulls = "ignore";
+      if (strcmp(nulls, "ignore") != 0 && strcmp(nulls, "false") != 0 &&
+          strcmp(nulls, "true") != 0 && strcmp(nulls, "propagate") != 0) {
+        sb_free(&qcol); sb_free(&qres);
+        *error = strdup("rolling-any/all: nulls must be ignore, false, true, or propagate");
+        return -1;
+      }
+      const char *sql_func = strcmp(func, "any") == 0 ? "BOOL_OR" : "BOOL_AND";
+      if (strcmp(nulls, "propagate") == 0) {
+        sb_appendf(sb, "%s AS (SELECT *, CASE WHEN COUNT(*) OVER (ORDER BY _rn ROWS BETWEEN %d PRECEDING AND CURRENT ROW) > COUNT(%s) OVER (ORDER BY _rn ROWS BETWEEN %d PRECEDING AND CURRENT ROW) THEN NULL ELSE %s(%s) OVER (ORDER BY _rn ROWS BETWEEN %d PRECEDING AND CURRENT ROW) END AS %s FROM %s)",
+                   cte_name, preceding, qcol.data, preceding, sql_func, qcol.data, preceding, qres.data, prev);
+      } else if (strcmp(nulls, "false") == 0 || strcmp(nulls, "true") == 0) {
+        const char *fill = strcmp(nulls, "true") == 0 ? "TRUE" : "FALSE";
+        sb_appendf(sb, "%s AS (SELECT *, %s(COALESCE(%s, %s)) OVER (ORDER BY _rn ROWS BETWEEN %d PRECEDING AND CURRENT ROW) AS %s FROM %s)",
+                   cte_name, sql_func, qcol.data, fill, preceding, qres.data, prev);
+      } else {
+        sb_appendf(sb, "%s AS (SELECT *, %s(%s) OVER (ORDER BY _rn ROWS BETWEEN %d PRECEDING AND CURRENT ROW) AS %s FROM %s)",
+                   cte_name, sql_func, qcol.data, preceding, qres.data, prev);
+      }
+      sb_free(&qcol); sb_free(&qres);
+      return 0;
+    }
+
+    const char *sql_func = "AVG";
+    if (strcmp(func, "sum") == 0) sql_func = "SUM";
+    else if (strcmp(func, "min") == 0) sql_func = "MIN";
+    else if (strcmp(func, "max") == 0) sql_func = "MAX";
+    else if (strcmp(func, "count") == 0) sql_func = "COUNT";
+    else if (strcmp(func, "avg") == 0 || strcmp(func, "mean") == 0) sql_func = "AVG";
+    sb_appendf(sb, "%s AS (SELECT * EXCLUDE (__tf_window_nn), "
+                 "CASE WHEN %s IS NULL THEN NULL ELSE %s(%s) OVER "
+                 "(ORDER BY __tf_window_nn RANGE BETWEEN %d PRECEDING AND CURRENT ROW) END AS %s "
+                 "FROM (SELECT *, COUNT(%s) OVER "
+                 "(ORDER BY _rn ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) "
+                 "AS __tf_window_nn FROM %s) __tf_window_src)",
+               cte_name, qcol.data, sql_func, qcol.data, preceding,
+               qres.data, qcol.data, prev);
     sb_free(&qcol); sb_free(&qres);
     return 0;
   }
@@ -1044,25 +1523,123 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     sql_quote_ident(&qcol, column);
     if (result) sql_quote_ident(&qres, result);
     else sb_appendf(&qres, "\"%s_%s\"", func, column);
-    sb_appendf(sb, "%s AS (SELECT *, %s(%s) OVER (ORDER BY rowid() ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS %s FROM %s)",
+    sb_appendf(sb, "%s AS (SELECT *, %s(%s) OVER (ORDER BY _rn ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS %s FROM %s)",
                cte_name, sql_func, qcol.data, qres.data, prev);
     sb_free(&qcol); sb_free(&qres);
     return 0;
   }
 
-  /* ---- lead ---- */
-  if (strcmp(op, "lead") == 0) {
+  /* ---- rowid ---- */
+  if (strcmp(op, "rowid") == 0) {
+    cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
+    const char *result = jstr(args, "result");
+    if (!result) result = "_rowid";
+
+    strbuf qres;
+    sb_init(&qres);
+    sql_quote_ident(&qres, result);
+
+    strbuf partition;
+    sb_init(&partition);
+    if (cJSON_IsArray(cols) && cJSON_GetArraySize(cols) > 0) {
+      int n_cols = cJSON_GetArraySize(cols);
+      for (int i = 0; i < n_cols; i++) {
+        cJSON *item = cJSON_GetArrayItem(cols, i);
+        if (!cJSON_IsString(item)) {
+          sb_free(&qres);
+          sb_free(&partition);
+          *error = strdup("rowid: columns must be strings");
+          return -1;
+        }
+        strbuf qcol;
+        sb_init(&qcol);
+        sql_quote_ident(&qcol, item->valuestring);
+        if (i > 0) sb_append(&partition, ", ");
+        sb_append(&partition, qcol.data);
+        sb_free(&qcol);
+      }
+    }
+
+    if (partition.len > 0) {
+      sb_appendf(sb, "%s AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY %s ORDER BY _rn) AS %s FROM %s)",
+                 cte_name, partition.data, qres.data, prev);
+    } else {
+      sb_appendf(sb, "%s AS (SELECT *, ROW_NUMBER() OVER (ORDER BY _rn) AS %s FROM %s)",
+                 cte_name, qres.data, prev);
+    }
+    sb_free(&qres);
+    sb_free(&partition);
+    return 0;
+  }
+
+  /* ---- rleid ---- */
+  if (strcmp(op, "rleid") == 0) {
+    cJSON *cols = cJSON_GetObjectItemCaseSensitive(args, "columns");
+    const char *result = jstr(args, "result");
+    if (!result) result = "_rleid";
+    if (!cJSON_IsArray(cols) || cJSON_GetArraySize(cols) <= 0) {
+      *error = strdup("rleid: missing 'columns'");
+      return -1;
+    }
+
+    strbuf qres;
+    sb_init(&qres);
+    sql_quote_ident(&qres, result);
+
+    strbuf change;
+    sb_init(&change);
+    int n_cols = cJSON_GetArraySize(cols);
+    for (int i = 0; i < n_cols; i++) {
+      cJSON *item = cJSON_GetArrayItem(cols, i);
+      if (!cJSON_IsString(item)) {
+        sb_free(&qres);
+        sb_free(&change);
+        *error = strdup("rleid: columns must be strings");
+        return -1;
+      }
+      strbuf qcol;
+      sb_init(&qcol);
+      sql_quote_ident(&qcol, item->valuestring);
+      if (i > 0) sb_append(&change, " OR ");
+      sb_appendf(&change, "%s IS DISTINCT FROM LAG(%s) OVER (ORDER BY _rn)",
+                 qcol.data, qcol.data);
+      sb_free(&qcol);
+    }
+
+    sb_appendf(sb, "%s AS (SELECT * EXCLUDE (__tf_rleid_changed, __tf_rleid_ord), "
+                   "SUM(__tf_rleid_changed) OVER (ORDER BY __tf_rleid_ord ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS %s "
+                   "FROM (SELECT *, _rn AS __tf_rleid_ord, "
+                   "CASE WHEN ROW_NUMBER() OVER (ORDER BY _rn) = 1 OR %s THEN 1 ELSE 0 END AS __tf_rleid_changed "
+                   "FROM %s) __tf_rleid_src)",
+               cte_name, qres.data, change.data, prev);
+    sb_free(&qres);
+    sb_free(&change);
+    return 0;
+  }
+
+  /* ---- lead / lag / shift ---- */
+  if (strcmp(op, "lead") == 0 || strcmp(op, "lag") == 0 || strcmp(op, "shift") == 0) {
     const char *column = jstr(args, "column");
     int offset = jint(args, "offset", 1);
     const char *result = jstr(args, "result");
-    if (!column) { *error = strdup("lead: missing 'column'"); return -1; }
+    const char *type = jstr(args, "type");
+    const char *fn = "LAG";
+    const char *suffix = "lag";
+    if (strcmp(op, "lead") == 0 || (strcmp(op, "shift") == 0 && type && strcmp(type, "lead") == 0)) {
+      fn = "LEAD";
+      suffix = "lead";
+    } else if (strcmp(op, "shift") == 0) {
+      suffix = "shift";
+    }
+    if (!column) { *error = strdup("shift: missing 'column'"); return -1; }
+    if (offset <= 0) offset = 1;
     strbuf qcol, qres;
     sb_init(&qcol); sb_init(&qres);
     sql_quote_ident(&qcol, column);
     if (result) sql_quote_ident(&qres, result);
-    else sb_appendf(&qres, "\"%s_lead_%d\"", column, offset);
-    sb_appendf(sb, "%s AS (SELECT *, LEAD(%s, %d) OVER (ORDER BY rowid()) AS %s FROM %s)",
-               cte_name, qcol.data, offset, qres.data, prev);
+    else sb_appendf(&qres, "\"%s_%s\"", column, suffix);
+    sb_appendf(sb, "%s AS (SELECT *, %s(%s, %d) OVER (ORDER BY _rn) AS %s FROM %s)",
+               cte_name, fn, qcol.data, offset, qres.data, prev);
     sb_free(&qcol); sb_free(&qres);
     return 0;
   }
@@ -1083,10 +1660,8 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
         cJSON *part = cJSON_GetArrayItem(extract, i);
         if (!cJSON_IsString(part)) continue;
         const char *p = part->valuestring;
-        char result_name[128];
-        snprintf(result_name, sizeof(result_name), "%s_%s", column, p);
         sb_appendf(&der, ", EXTRACT(%s FROM %s::TIMESTAMP) AS ", p, qcol.data);
-        sql_quote_ident(&der, result_name);
+        sql_quote_joined_ident(&der, column, "_", p);
       }
     }
     sb_appendf(sb, "%s AS (SELECT *%s FROM %s)", cte_name, der.data, prev);
@@ -1105,23 +1680,23 @@ static int emit_cte(strbuf *sb, const char *cte_name, const char *prev,
     sql_quote_ident(&qcol, column);
     strbuf qres;
     sb_init(&qres);
-    if (result) sql_quote_ident(&qres, result);
-    else {
-      char buf[128];
-      snprintf(buf, sizeof(buf), "%s_%s", column, trunc);
-      sql_quote_ident(&qres, buf);
+    int in_place = result == NULL;
+    sql_quote_ident(&qres, result ? result : column);
+    if (in_place) {
+      sb_appendf(sb, "%s AS (SELECT * REPLACE (date_trunc('%s', %s::TIMESTAMP) AS %s) FROM %s)",
+                 cte_name, trunc, qcol.data, qres.data, prev);
+    } else {
+      sb_appendf(sb, "%s AS (SELECT *, date_trunc('%s', %s::TIMESTAMP) AS %s FROM %s)",
+                 cte_name, trunc, qcol.data, qres.data, prev);
     }
-    sb_appendf(sb, "%s AS (SELECT *, date_trunc('%s', %s::TIMESTAMP) AS %s FROM %s)",
-               cte_name, trunc, qcol.data, qres.data, prev);
     sb_free(&qcol); sb_free(&qres);
     return 0;
   }
 
   /* ---- stats ---- */
   if (strcmp(op, "stats") == 0) {
-    /* Stats uses DuckDB's SUMMARIZE — works without knowing column names */
-    sb_appendf(sb, "%s AS (SELECT * FROM (SUMMARIZE SELECT * FROM %s))", cte_name, prev);
-    return 0;
+    *error = strdup("stats: native report shape cannot be lowered to SQL");
+    return -1;
   }
 
   /* ---- flatten ---- */
@@ -1174,7 +1749,25 @@ char *tf_ir_to_sql(const tf_ir_plan *plan, char **error) {
   /* No transforms — just SELECT * */
   if (first_transform >= last_transform) {
     sb_appendf(&sb, "SELECT * FROM %s", input_source);
-    return sb_detach(&sb);
+    char *sql = sb_detach(&sb);
+    if (!sql && error && !*error) *error = strdup("sql: out of memory");
+    return sql;
+  }
+
+  /* Check if any operator needs row ordering (_rn column) */
+  int needs_rn = 0;
+  for (size_t i = first_transform; i < last_transform; i++) {
+    const char *op = plan->nodes[i].op;
+    if (strcmp(op, "window") == 0 || strcmp(op, "rolling-sum") == 0 ||
+        strcmp(op, "rolling-mean") == 0 || strcmp(op, "rolling-min") == 0 ||
+        strcmp(op, "rolling-max") == 0 || strcmp(op, "rolling-any") == 0 ||
+        strcmp(op, "rolling-all") == 0 || strcmp(op, "step") == 0 ||
+        strcmp(op, "lead") == 0 || strcmp(op, "lag") == 0 ||
+        strcmp(op, "shift") == 0 || strcmp(op, "rowid") == 0 ||
+        strcmp(op, "rleid") == 0 || strcmp(op, "fill-down") == 0) {
+      needs_rn = 1;
+      break;
+    }
   }
 
   /* Build CTE chain — use two alternating name buffers so prev != current */
@@ -1185,6 +1778,13 @@ char *tf_ir_to_sql(const tf_ir_plan *plan, char **error) {
 
   sb_append(&sb, "WITH\n");
 
+  /* If row ordering is needed, inject a _rn column from the source */
+  if (needs_rn) {
+    sb_appendf(&sb, "  _numbered AS (SELECT *, ROW_NUMBER() OVER () AS _rn FROM %s)", input_source);
+    prev = "_numbered";
+    n_ctes++;
+  }
+
   for (size_t i = first_transform; i < last_transform; i++) {
     const tf_ir_node *node = &plan->nodes[i];
     char *cte_name = name_bufs[n_ctes % 2];
@@ -1199,13 +1799,24 @@ char *tf_ir_to_sql(const tf_ir_plan *plan, char **error) {
       else free(err);
       return NULL;
     }
+    if (sb.failed) {
+      sb_free(&sb);
+      if (error) *error = strdup("sql: out of memory");
+      return NULL;
+    }
 
     prev = cte_name;
     n_ctes++;
   }
 
-  /* Final SELECT from last CTE */
-  sb_appendf(&sb, "\nSELECT * FROM %s", prev);
+  /* Final SELECT from last CTE — exclude internal _rn column */
+  if (needs_rn) {
+    sb_appendf(&sb, "\nSELECT * EXCLUDE (_rn) FROM %s", prev);
+  } else {
+    sb_appendf(&sb, "\nSELECT * FROM %s", prev);
+  }
 
-  return sb_detach(&sb);
+  char *sql = sb_detach(&sb);
+  if (!sql && error && !*error) *error = strdup("sql: out of memory");
+  return sql;
 }