tranfi 0.0.2 → 0.2.0

package/csrc/codec_jsonl.c

@@ -0,0 +1,1086 @@
+/*
+ * codec_jsonl.c — JSON Lines streaming decoder and encoder.
+ *
+ * Decoder: splits on newlines, parses each line as JSON object using cJSON.
+ *   - First line establishes schema (column names and types).
+ *   - Subsequent lines match schema; missing keys → null.
+ *   - Emits batch at batch_size.
+ *
+ * Encoder: batch → one JSON object per row, newline-separated.
+ */
+
+#include "internal.h"
+#include "date_utils.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+
+#define DEFAULT_BATCH_SIZE 1024
+#define DEFAULT_MAX_ERROR_BYTES 4096
+#define DEFAULT_MAX_RECORD_BYTES (64 * 1024 * 1024)
+
+/* ================================================================
+ * JSONL Decoder
+ * ================================================================ */
+
+typedef enum {
+    JSONL_ERROR_SKIP = 0,
+    JSONL_ERROR_FAIL,
+    JSONL_ERROR_WARN,
+    JSONL_ERROR_QUARANTINE,
+} jsonl_error_action;
+
+static const char *jsonl_error_action_name(jsonl_error_action action) {
+    switch (action) {
+        case JSONL_ERROR_FAIL: return "fail";
+        case JSONL_ERROR_WARN: return "warn";
+        case JSONL_ERROR_QUARANTINE: return "quarantine";
+        case JSONL_ERROR_SKIP:
+        default: return "skip";
+    }
+}
+
+static int parse_jsonl_error_action(const char *s, jsonl_error_action *out) {
+    if (!s || strcmp(s, "skip") == 0 || strcmp(s, "ignore") == 0) {
+        *out = JSONL_ERROR_SKIP;
+        return 1;
+    }
+    if (strcmp(s, "fail") == 0 || strcmp(s, "error") == 0 || strcmp(s, "stop") == 0) {
+        *out = JSONL_ERROR_FAIL;
+        return 1;
+    }
+    if (strcmp(s, "warn") == 0 || strcmp(s, "warning") == 0) {
+        *out = JSONL_ERROR_WARN;
+        return 1;
+    }
+    if (strcmp(s, "quarantine") == 0 || strcmp(s, "dead-letter") == 0 || strcmp(s, "dead_letter") == 0) {
+        *out = JSONL_ERROR_QUARANTINE;
+        return 1;
+    }
+    return 0;
+}
+
+typedef struct {
+    const char *name;
+    size_t column;
+} jsonl_field;
+
+typedef struct {
+    size_t    batch_size;
+    jsonl_error_action on_error;
+    size_t    max_error_bytes;
+    size_t    max_record_bytes;
+    size_t    line_number;
+    size_t    byte_offset;
+    size_t    bad_records;
+    int       pending_lf; /* A CR at the end of the previous input chunk. */
+
+    /* Line accumulator */
+    tf_buffer line_buf;
+
+    /* Schema (discovered from first object) */
+    char    **col_names;
+    tf_type  *col_types;
+    size_t    n_cols;
+    int       schema_ready;
+    jsonl_field *fields;  /* Sorted names, with original order breaking duplicate ties. */
+    size_t *aliases;
+    cJSON **values;       /* Borrowed from the current parsed record only. */
+
+    /* Current batch */
+    tf_batch *batch;
+    size_t    rows_buffered;
+} jsonl_decoder_state;
+
+static tf_type json_to_type(const cJSON *val) {
+    if (cJSON_IsNumber(val)) {
+        /* Check if integer */
+        double d = val->valuedouble;
+        if (d >= -9007199254740992.0 && d <= 9007199254740992.0 && d == (double)(int64_t)d)
+            return TF_TYPE_INT64;
+        return TF_TYPE_FLOAT64;
+    }
+    if (cJSON_IsString(val)) return TF_TYPE_STRING;
+    if (cJSON_IsBool(val)) return TF_TYPE_BOOL;
+    if (cJSON_IsArray(val) || cJSON_IsObject(val)) return TF_TYPE_STRING;
+    return TF_TYPE_NULL;
+}
+
+static tf_type widen_type(tf_type current, tf_type incoming) {
+    if (current == incoming) return current;
+    if (current == TF_TYPE_NULL) return incoming;
+    if (incoming == TF_TYPE_NULL) return current;
+    if (current == TF_TYPE_INT64 && incoming == TF_TYPE_FLOAT64) return TF_TYPE_FLOAT64;
+    if (current == TF_TYPE_FLOAT64 && incoming == TF_TYPE_INT64) return TF_TYPE_FLOAT64;
+    return TF_TYPE_STRING;
+}
+
+static size_t jsonl_type_size(tf_type t) {
+    switch (t) {
+        case TF_TYPE_BOOL: return sizeof(uint8_t);
+        case TF_TYPE_INT64: return sizeof(int64_t);
+        case TF_TYPE_FLOAT64: return sizeof(double);
+        case TF_TYPE_STRING: return sizeof(char *);
+        case TF_TYPE_DATE: return sizeof(int32_t);
+        case TF_TYPE_TIMESTAMP: return sizeof(int64_t);
+        default: return 0;
+    }
+}
+
+static void jsonl_free_schema_arrays(char **col_names, tf_type *col_types, size_t n_names) {
+    if (col_names) {
+        for (size_t i = 0; i < n_names; i++) free(col_names[i]);
+    }
+    free(col_names);
+    free(col_types);
+}
+
+static int jsonl_convert_batch_column(tf_batch *batch, size_t col, tf_type new_type) {
+    if (!batch || col >= batch->n_cols || batch->col_types[col] == new_type) return TF_OK;
+
+    tf_type old_type = batch->col_types[col];
+    size_t sz = jsonl_type_size(new_type);
+    void *new_col = NULL;
+    uint8_t *new_nulls = NULL;
+    if (sz == 0) {
+        batch->col_types[col] = new_type;
+        batch->columns[col] = NULL;
+        batch->nulls[col] = NULL;
+        return TF_OK;
+    }
+
+    size_t column_bytes = 0;
+    if (tf_size_mul(sz, batch->capacity, &column_bytes) != TF_OK) return TF_ERROR;
+    new_col = tf_arena_alloc(batch->arena, column_bytes);
+    new_nulls = tf_arena_alloc(batch->arena, batch->capacity);
+    if (!new_col || !new_nulls) return TF_ERROR;
+    memset(new_nulls, 1, batch->capacity);
+
+    for (size_t r = 0; r < batch->n_rows; r++) {
+        int was_null = (!batch->nulls[col] || batch->nulls[col][r] != 0);
+        if (was_null) continue;
+
+        if (new_type == TF_TYPE_FLOAT64 && old_type == TF_TYPE_INT64) {
+            ((double *)new_col)[r] = (double)((int64_t *)batch->columns[col])[r];
+            new_nulls[r] = 0;
+        } else if (new_type == TF_TYPE_FLOAT64 && old_type == TF_TYPE_FLOAT64) {
+            ((double *)new_col)[r] = ((double *)batch->columns[col])[r];
+            new_nulls[r] = 0;
+        } else if (new_type == TF_TYPE_STRING) {
+            char numbuf[64];
+            const char *s = NULL;
+            switch (old_type) {
+                case TF_TYPE_BOOL:
+                    s = ((uint8_t *)batch->columns[col])[r] ? "true" : "false";
+                    break;
+                case TF_TYPE_INT64:
+                    snprintf(numbuf, sizeof(numbuf), "%lld",
+                             (long long)((int64_t *)batch->columns[col])[r]);
+                    s = numbuf;
+                    break;
+                case TF_TYPE_FLOAT64:
+                    if (tf_format_float64(numbuf, sizeof(numbuf),
+                                          ((double *)batch->columns[col])[r]) != TF_OK)
+                        return TF_ERROR;
+                    s = numbuf;
+                    break;
+                case TF_TYPE_STRING:
+                    s = ((char **)batch->columns[col])[r];
+                    break;
+                default:
+                    s = "";
+                    break;
+            }
+            size_t len = 0, bytes = 0;
+            const char *safe_s = s ? s : "";
+            if (tf_string_length_bounded(safe_s, TF_MAX_CELL_BYTES,
+                                         &len, "jsonl", "cell") != TF_OK ||
+                tf_size_add(len, 1, &bytes) != TF_OK)
+                return TF_ERROR;
+            char *copy = tf_arena_alloc(batch->arena, bytes);
+            if (!copy) return TF_ERROR;
+            memcpy(copy, safe_s, len);
+            copy[len] = '\0';
+            ((char **)new_col)[r] = copy;
+            new_nulls[r] = 0;
+        }
+    }
+
+    batch->col_types[col] = new_type;
+    batch->columns[col] = new_col;
+    batch->nulls[col] = new_nulls;
+    return TF_OK;
+}
+
+static int jsonl_widen_column(jsonl_decoder_state *st, size_t col, tf_type incoming) {
+    tf_type widened = widen_type(st->col_types[col], incoming);
+    if (widened == st->col_types[col]) return TF_OK;
+    if (jsonl_convert_batch_column(st->batch, col, widened) != TF_OK) return TF_ERROR;
+    st->col_types[col] = widened;
+    return TF_OK;
+}
+
+static int emit_jsonl_malformed(jsonl_decoder_state *st, const char *line, size_t len,
+                                size_t line_no, const char *reason,
+                                tf_side_channels *side) {
+    st->bad_records++;
+    if (!side || !side->errors) return TF_OK;
+
+    size_t keep = len;
+    int truncated = 0;
+    if (st->max_error_bytes > 0 && keep > st->max_error_bytes) {
+        keep = st->max_error_bytes;
+        truncated = 1;
+    }
+    char *raw = malloc(keep + 1);
+    if (!raw) return TF_ERROR;
+    memcpy(raw, line, keep);
+    raw[keep] = '\0';
+
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) { free(raw); return TF_ERROR; }
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "jsonl_malformed") != TF_OK ||
+        tf_json_add_string(obj, "op", "codec.jsonl.decode") != TF_OK ||
+        tf_json_add_string(obj, "action", jsonl_error_action_name(st->on_error)) != TF_OK ||
+        tf_json_add_string(obj, "severity", st->on_error == JSONL_ERROR_WARN ? "warning" : "error") != TF_OK ||
+        tf_json_add_number(obj, "line", (double)line_no) != TF_OK ||
+        tf_json_add_string(obj, "message", reason ? reason : "invalid JSONL record") != TF_OK ||
+        tf_json_add_number(obj, "raw_bytes", (double)len) != TF_OK ||
+        tf_json_add_string(obj, "raw", raw) != TF_OK ||
+        (truncated && tf_json_add_bool(obj, "truncated", 1) != TF_OK)) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->errors, obj);
+done:
+    cJSON_Delete(obj);
+    free(raw);
+    return rc;
+}
+
+static int handle_jsonl_malformed(jsonl_decoder_state *st, const char *line, size_t len,
+                                  size_t line_no, const char *reason,
+                                  tf_side_channels *side) {
+    if (st->on_error == JSONL_ERROR_SKIP) {
+        st->bad_records++;
+        return TF_OK;
+    }
+    if (emit_jsonl_malformed(st, line, len, line_no, reason, side) != TF_OK)
+        return TF_ERROR;
+    if (st->on_error == JSONL_ERROR_FAIL) {
+        char msg[256];
+        snprintf(msg, sizeof(msg), "jsonl decode failed at line %zu: %s",
+                 line_no, reason ? reason : "invalid JSONL record");
+        tf_set_last_error(msg);
+        return TF_ERROR;
+    }
+    return TF_OK;
+}
+
+static char *jsonl_record_preview(jsonl_decoder_state *st,
+                                  const uint8_t *prefix, size_t prefix_len,
+                                  const uint8_t *suffix, size_t suffix_len,
+                                  int *truncated_out) {
+    size_t total = prefix_len + suffix_len;
+    size_t keep = total;
+    int truncated = 0;
+    if (st->max_error_bytes > 0 && keep > st->max_error_bytes) {
+        keep = st->max_error_bytes;
+        truncated = 1;
+    }
+    char *raw = malloc(keep + 1);
+    if (!raw) return NULL;
+    size_t copied = 0;
+    size_t n = prefix_len < keep ? prefix_len : keep;
+    if (n > 0 && prefix) {
+        memcpy(raw, prefix, n);
+        copied += n;
+    }
+    if (copied < keep && suffix) {
+        size_t m = keep - copied;
+        if (m > suffix_len) m = suffix_len;
+        if (m > 0) {
+            memcpy(raw + copied, suffix, m);
+            copied += m;
+        }
+    }
+    raw[copied] = '\0';
+    if (truncated_out) *truncated_out = truncated;
+    return raw;
+}
+
+static int emit_jsonl_record_size_diagnostic(jsonl_decoder_state *st,
+                                             const uint8_t *prefix, size_t prefix_len,
+                                             const uint8_t *suffix, size_t suffix_len,
+                                             size_t line_no, size_t byte_offset,
+                                             size_t observed_len,
+                                             tf_side_channels *side) {
+    if (!side || !side->errors) return TF_OK;
+    int truncated = 0;
+    char *raw = jsonl_record_preview(st, prefix, prefix_len, suffix, suffix_len, &truncated);
+    if (!raw) return TF_ERROR;
+
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) { free(raw); return TF_ERROR; }
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "jsonl_record_too_large") != TF_OK ||
+        tf_json_add_string(obj, "op", "codec.jsonl.decode") != TF_OK ||
+        tf_json_add_string(obj, "action", "fail") != TF_OK ||
+        tf_json_add_string(obj, "severity", "error") != TF_OK ||
+        tf_json_add_number(obj, "line", (double)line_no) != TF_OK ||
+        tf_json_add_number(obj, "byte_offset", (double)byte_offset) != TF_OK ||
+        tf_json_add_number(obj, "max_record_bytes", (double)st->max_record_bytes) != TF_OK ||
+        tf_json_add_number(obj, "observed_bytes", (double)observed_len) != TF_OK ||
+        tf_json_add_string(obj, "message", "JSONL record exceeds max_record_bytes") != TF_OK ||
+        tf_json_add_number(obj, "raw_bytes", (double)observed_len) != TF_OK ||
+        tf_json_add_string(obj, "raw", raw) != TF_OK ||
+        (truncated && tf_json_add_bool(obj, "truncated", 1) != TF_OK)) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->errors, obj);
+done:
+    cJSON_Delete(obj);
+    free(raw);
+    return rc;
+}
+
+static int fail_jsonl_record_limit(jsonl_decoder_state *st,
+                                   const uint8_t *prefix, size_t prefix_len,
+                                   const uint8_t *suffix, size_t suffix_len,
+                                   size_t line_no, size_t byte_offset,
+                                   size_t observed_len,
+                                   tf_side_channels *side) {
+    if (emit_jsonl_record_size_diagnostic(st, prefix, prefix_len, suffix, suffix_len,
+                                          line_no, byte_offset, observed_len, side) != TF_OK)
+        return TF_ERROR;
+    char err[256];
+    snprintf(err, sizeof(err),
+             "jsonl record exceeds max_record_bytes at line %zu: max %zu bytes, observed %zu bytes",
+             line_no, st->max_record_bytes, observed_len);
+    tf_set_last_error(err);
+    return TF_ERROR;
+}
+
+static int check_jsonl_incoming_record_limit(jsonl_decoder_state *st,
+                                             const uint8_t *data, size_t len,
+                                             tf_side_channels *side) {
+    if (st->max_record_bytes == 0 || len == 0) return TF_OK;
+
+    size_t prefix_len = tf_buffer_readable(&st->line_buf);
+    const uint8_t *prefix = prefix_len > 0 ? st->line_buf.data + st->line_buf.read_pos : NULL;
+    size_t incoming_base = st->byte_offset + prefix_len;
+    size_t line_no = st->line_number + 1;
+    size_t line_offset = st->byte_offset;
+    size_t segment_start = 0;
+    size_t current_len = prefix_len;
+
+    for (size_t i = 0; i < len; i++) {
+        if (data[i] == '\n' || data[i] == '\r') {
+            if (data[i] == '\r' && i + 1 < len && data[i + 1] == '\n') i++;
+            prefix = NULL;
+            prefix_len = 0;
+            current_len = 0;
+            segment_start = i + 1;
+            line_offset = incoming_base + i + 1;
+            line_no++;
+            continue;
+        }
+        current_len++;
+        if (current_len > st->max_record_bytes) {
+            size_t suffix_len = i - segment_start + 1;
+            return fail_jsonl_record_limit(st, prefix, prefix_len,
+                                           data + segment_start, suffix_len,
+                                           line_no, line_offset, current_len, side);
+        }
+    }
+    return TF_OK;
+}
+
+static int check_jsonl_buffer_record_limit(jsonl_decoder_state *st, size_t observed_len,
+                                           size_t line_no, size_t byte_offset,
+                                           tf_side_channels *side) {
+    if (st->max_record_bytes == 0 || observed_len <= st->max_record_bytes) return TF_OK;
+    const uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+    return fail_jsonl_record_limit(st, buf, observed_len, NULL, 0,
+                                   line_no, byte_offset, observed_len, side);
+}
+
+static int emit_jsonl_batch(tf_batch *batch, tf_batch ***out, size_t *n_out, size_t *out_cap) {
+    if (*n_out >= *out_cap) {
+        size_t need = 0;
+        size_t new_cap = 0;
+        if (tf_size_add(*n_out, 1, &need) != TF_OK ||
+            tf_size_grow_pow2(*out_cap, need, 4, &new_cap) != TF_OK) {
+            return TF_ERROR;
+        }
+        tf_batch **tmp = tf_reallocarray_checked(*out, new_cap, sizeof(tf_batch *));
+        if (!tmp) return TF_ERROR;
+        *out = tmp;
+        *out_cap = new_cap;
+    }
+    (*out)[(*n_out)++] = batch;
+    return TF_OK;
+}
+
+static tf_batch *make_jsonl_batch(jsonl_decoder_state *st) {
+    tf_batch *b = tf_batch_create(st->n_cols, st->batch_size);
+    if (!b) return NULL;
+    for (size_t i = 0; i < st->n_cols; i++) {
+        if (tf_batch_set_schema(b, i, st->col_names[i], st->col_types[i]) != TF_OK) {
+            tf_batch_free(b);
+            return NULL;
+        }
+    }
+    return b;
+}
+
+static int jsonl_field_compare(const void *a, const void *b) {
+    const jsonl_field *x = a, *y = b;
+    int cmp = strcmp(x->name, y->name);
+    return cmp ? cmp : (x->column > y->column) - (x->column < y->column);
+}
+
+static size_t jsonl_find_column(const jsonl_decoder_state *st, const char *name) {
+    size_t lo = 0, hi = st->n_cols;
+    while (lo < hi) {
+        size_t mid = lo + (hi - lo) / 2;
+        if (strcmp(st->fields[mid].name, name) < 0) lo = mid + 1;
+        else hi = mid;
+    }
+    return lo < st->n_cols && strcmp(st->fields[lo].name, name) == 0
+        ? st->fields[lo].column : st->n_cols;
+}
+
+static int jsonl_index_schema(jsonl_decoder_state *st) {
+    size_t n = st->n_cols ? st->n_cols : 1;
+    st->fields = tf_callocarray_checked(n, sizeof(*st->fields));
+    st->aliases = tf_callocarray_checked(n, sizeof(*st->aliases));
+    st->values = tf_callocarray_checked(n, sizeof(*st->values));
+    if (!st->fields || !st->aliases || !st->values) return TF_ERROR;
+    for (size_t c = 0; c < st->n_cols; c++)
+        st->fields[c] = (jsonl_field){st->col_names[c], c};
+    qsort(st->fields, st->n_cols, sizeof(*st->fields), jsonl_field_compare);
+    for (size_t c = 0; c < st->n_cols; c++)
+        st->aliases[c] = jsonl_find_column(st, st->col_names[c]);
+    return TF_OK;
+}
+
+static int add_json_row(jsonl_decoder_state *st) {
+    if (!st->batch) {
+        st->batch = make_jsonl_batch(st);
+        if (!st->batch) return TF_ERROR;
+    }
+
+    size_t row = st->batch->n_rows;
+    size_t need_rows = 0;
+    if (tf_size_add(row, 1, &need_rows) != TF_OK ||
+        tf_batch_ensure_capacity(st->batch, need_rows) != TF_OK)
+        return TF_ERROR;
+
+    for (size_t c = 0; c < st->n_cols; c++) {
+        cJSON *val = st->values[st->aliases[c]];
+        int rc = TF_OK;
+        if (!val || cJSON_IsNull(val)) {
+            rc = tf_batch_set_null(st->batch, row, c);
+        } else {
+            switch (st->col_types[c]) {
+                case TF_TYPE_BOOL:
+                    rc = tf_batch_set_bool(st->batch, row, c, cJSON_IsTrue(val));
+                    break;
+                case TF_TYPE_INT64:
+                    rc = tf_batch_set_int64(st->batch, row, c, (int64_t)val->valuedouble);
+                    break;
+                case TF_TYPE_FLOAT64:
+                    rc = tf_batch_set_float64(st->batch, row, c, val->valuedouble);
+                    break;
+                case TF_TYPE_STRING:
+                    if (cJSON_IsString(val)) {
+                        rc = tf_batch_set_string(st->batch, row, c, val->valuestring ? val->valuestring : "");
+                    } else {
+                        char *printed = cJSON_PrintUnformatted(val);
+                        if (!printed) return TF_ERROR;
+                        rc = tf_batch_set_string(st->batch, row, c, printed);
+                        free(printed);
+                    }
+                    break;
+                default:
+                    rc = tf_batch_set_null(st->batch, row, c);
+                    break;
+            }
+        }
+        if (rc != TF_OK) return TF_ERROR;
+    }
+    if (tf_batch_expose_row(st->batch, row) != TF_OK) return TF_ERROR;
+    st->rows_buffered++;
+    return TF_OK;
+}
+
+
+static int jsonl_space(unsigned char c) {
+    return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+}
+
+/* cJSON owns structural parsing. Check lexemes it deliberately accepts more
+ * loosely, before its C strings can lose embedded NULs. */
+static int jsonl_valid_lexemes(const char *text, size_t n) {
+    const unsigned char *s = (const unsigned char *)text;
+    for (size_t i = 0; i < n;) {
+        unsigned char c = s[i++];
+        if (c == '"') {
+            int closed = 0;
+            while (i < n) {
+                c = s[i++];
+                if (c == '"') { closed = 1; break; }
+                if (c < 0x20) return 0;
+                if (c == '\\') {
+                    if (i == n) return 0;
+                    c = s[i++];
+                    if (c == 'u') {
+                        if (n - i < 4) return 0;
+                        unsigned value = 0;
+                        for (size_t k = 0; k < 4; k++) {
+                            unsigned char h = s[i++];
+                            if (h >= '0' && h <= '9') value = value * 16 + h - '0';
+                            else if (h >= 'a' && h <= 'f') value = value * 16 + h - 'a' + 10;
+                            else if (h >= 'A' && h <= 'F') value = value * 16 + h - 'A' + 10;
+                            else return 0;
+                        }
+                        if (!value) return 0;
+                    } else if (!strchr("\"\\/bfnrt", c)) return 0;
+                } else if (c >= 0x80) {
+                    unsigned value, minimum;
+                    size_t extra;
+                    if (c >= 0xc2 && c <= 0xdf) { value = c & 31; extra = 1; minimum = 0x80; }
+                    else if (c >= 0xe0 && c <= 0xef) { value = c & 15; extra = 2; minimum = 0x800; }
+                    else if (c >= 0xf0 && c <= 0xf4) { value = c & 7; extra = 3; minimum = 0x10000; }
+                    else return 0;
+                    if (n - i < extra) return 0;
+                    while (extra--) {
+                        if ((s[i] & 0xc0) != 0x80) return 0;
+                        value = (value << 6) | (s[i++] & 63);
+                    }
+                    if (value < minimum || value > 0x10ffff ||
+                        (value >= 0xd800 && value <= 0xdfff)) return 0;
+                }
+            }
+            if (!closed) return 0;
+        } else if (c == '-' || (c >= '0' && c <= '9')) {
+            i--;
+            if (s[i] == '-' && ++i == n) return 0;
+            if (s[i] == '0') i++;
+            else {
+                if (s[i] < '1' || s[i] > '9') return 0;
+                do { i++; } while (i < n && s[i] >= '0' && s[i] <= '9');
+            }
+            if (i < n && s[i] == '.') {
+                i++;
+                size_t start = i;
+                while (i < n && s[i] >= '0' && s[i] <= '9') i++;
+                if (i == start) return 0;
+            }
+            if (i < n && (s[i] == 'e' || s[i] == 'E')) {
+                i++;
+                if (i < n && (s[i] == '+' || s[i] == '-')) i++;
+                size_t start = i;
+                while (i < n && s[i] >= '0' && s[i] <= '9') i++;
+                if (i == start) return 0;
+            }
+            if (i < n && !jsonl_space(s[i]) && s[i] != ',' && s[i] != ']' && s[i] != '}')
+                return 0;
+        } else if (c >= 0x80 || (c < 0x20 && !jsonl_space(c))) return 0;
+    }
+    return 1;
+}
+
+static int jsonl_finite_values(const cJSON *obj) {
+    for (const cJSON *v = obj; v; v = v->next) {
+        if (cJSON_IsNumber(v) && !isfinite(v->valuedouble)) return 0;
+        if (v->child && !jsonl_finite_values(v->child)) return 0;
+    }
+    return 1;
+}
+
+static int process_jsonl_line(jsonl_decoder_state *st, const char *line, size_t len,
+                              size_t line_no, tf_batch ***out, size_t *n_out,
+                              size_t *out_cap, tf_side_channels *side) {
+    /* Skip empty lines */
+    if (len == 0) return TF_OK;
+
+    if (!jsonl_valid_lexemes(line, len))
+        return handle_jsonl_malformed(st, line, len, line_no, "invalid JSON", side);
+    const char *end = NULL;
+    cJSON *obj = cJSON_ParseWithLengthOpts(line, len, &end, 0);
+    if (!obj && cJSON_ParseHadAllocationFailure()) {
+        return TF_ERROR;
+    }
+    while (end && end < line + len && jsonl_space((unsigned char)*end)) end++;
+    if (obj && (end != line + len || !jsonl_finite_values(obj))) {
+        cJSON_Delete(obj);
+        return handle_jsonl_malformed(st, line, len, line_no, "invalid JSON", side);
+    }
+    if (!obj || !cJSON_IsObject(obj)) {
+        const char *reason = obj ? "JSONL record is not an object" : "invalid JSON";
+        cJSON_Delete(obj);
+        return handle_jsonl_malformed(st, line, len, line_no, reason, side);
+    }
+
+    if (!st->schema_ready) {
+        /* Discover schema from first object */
+        int n = cJSON_GetArraySize(obj);
+        if (n < 0) { cJSON_Delete(obj); return TF_ERROR; }
+        size_t n_cols = (size_t)n;
+        if (n_cols > TF_MAX_COLUMNS) {
+            char err[256];
+            snprintf(err, sizeof(err),
+                     "jsonl schema exceeds max_columns: max %zu columns, observed %zu",
+                     TF_MAX_COLUMNS, n_cols);
+            tf_set_last_error(err);
+            cJSON_Delete(obj);
+            return TF_ERROR;
+        }
+        char **col_names = tf_callocarray_checked(n_cols ? n_cols : 1, sizeof(char *));
+        tf_type *col_types = tf_callocarray_checked(n_cols ? n_cols : 1, sizeof(tf_type));
+        if (!col_names || !col_types) {
+            free(col_names);
+            free(col_types);
+            cJSON_Delete(obj);
+            return TF_ERROR;
+        }
+
+        size_t i = 0;
+        size_t schema_bytes = 0;
+        cJSON *item;
+        cJSON_ArrayForEach(item, obj) {
+            const char *raw_name = item->string ? item->string : "";
+            size_t name_len = 0, name_bytes = 0, next_schema_bytes = 0;
+            if (tf_string_length_bounded(raw_name, TF_MAX_COLUMN_NAME_BYTES,
+                                         &name_len, "jsonl", "column name") != TF_OK ||
+                tf_size_add(name_len, 1, &name_bytes) != TF_OK ||
+                tf_size_add(schema_bytes, name_bytes, &next_schema_bytes) != TF_OK ||
+                tf_check_byte_limit(next_schema_bytes, TF_MAX_SCHEMA_BYTES,
+                                    "jsonl", "schema") != TF_OK) {
+                jsonl_free_schema_arrays(col_names, col_types, i);
+                cJSON_Delete(obj);
+                return TF_ERROR;
+            }
+            col_names[i] = malloc(name_bytes);
+            if (!col_names[i]) {
+                jsonl_free_schema_arrays(col_names, col_types, i);
+                cJSON_Delete(obj);
+                return TF_ERROR;
+            }
+            memcpy(col_names[i], raw_name, name_len);
+            col_names[i][name_len] = '\0';
+            col_types[i] = json_to_type(item);
+            schema_bytes = next_schema_bytes;
+            i++;
+        }
+        st->n_cols = n_cols;
+        st->col_names = col_names;
+        st->col_types = col_types;
+        if (jsonl_index_schema(st) != TF_OK) { cJSON_Delete(obj); return TF_ERROR; }
+        st->schema_ready = 1;
+    }
+    memset(st->values, 0, st->n_cols * sizeof(*st->values));
+    cJSON *item;
+    cJSON_ArrayForEach(item, obj) {
+        if (!item->string) continue;
+        size_t c = jsonl_find_column(st, item->string);
+        if (c == st->n_cols) continue;
+        /* cJSON lookup historically selects the first duplicate value. */
+        if (!st->values[c]) st->values[c] = item;
+        if (jsonl_widen_column(st, c, json_to_type(item)) != TF_OK) {
+            cJSON_Delete(obj);
+            return TF_ERROR;
+        }
+    }
+    for (size_t c = 0; c < st->n_cols; c++) {
+        cJSON *value = st->values[st->aliases[c]];
+        /* Duplicate schema names also receive that first value; ensure its
+         * numeric range is representable before a typed setter casts it. */
+        if (value && jsonl_widen_column(st, c, json_to_type(value)) != TF_OK) {
+            cJSON_Delete(obj);
+            return TF_ERROR;
+        }
+    }
+    int rc = add_json_row(st);
+    cJSON_Delete(obj);
+    if (rc != TF_OK) return rc;
+
+    /* Emit batch if full */
+    if (st->rows_buffered >= st->batch_size) {
+        if (emit_jsonl_batch(st->batch, out, n_out, out_cap) != TF_OK) return TF_ERROR;
+        st->batch = NULL;
+        st->rows_buffered = 0;
+    }
+
+    return TF_OK;
+}
+
+static int jsonl_decode(tf_decoder *self, const uint8_t *data, size_t len,
+                        tf_batch ***out, size_t *n_out, tf_side_channels *side) {
+    jsonl_decoder_state *st = self->state;
+    *out = NULL;
+    *n_out = 0;
+
+    if (len && st->pending_lf) {
+        st->pending_lf = 0;
+        if (data[0] == '\n') {
+            data++;
+            len--;
+            st->byte_offset++;
+        }
+    }
+    if (!len) return TF_OK;
+    if (check_jsonl_incoming_record_limit(st, data, len, side) != TF_OK) return TF_ERROR;
+    if (tf_buffer_write(&st->line_buf, data, len) != TF_OK) return TF_ERROR;
+
+    size_t out_cap = 0;
+    uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+    size_t buf_len = st->line_buf.len - st->line_buf.read_pos;
+
+    size_t line_start = 0;
+    for (size_t i = 0; i < buf_len; i++) {
+        if (buf[i] == '\n' || buf[i] == '\r') {
+            size_t line_len = i - line_start;
+            size_t line_no = ++st->line_number;
+            if (buf[i] == '\r') {
+                if (i + 1 < buf_len && buf[i + 1] == '\n') i++;
+                else if (i + 1 == buf_len) st->pending_lf = 1;
+            }
+            if (line_len > 0) {
+                if (process_jsonl_line(st, (const char *)buf + line_start, line_len,
+                                       line_no, out, n_out, &out_cap, side) != TF_OK)
+                    return TF_ERROR;
+            }
+            line_start = i + 1;
+        }
+    }
+
+    st->line_buf.read_pos += line_start;
+    st->byte_offset += line_start;
+    tf_buffer_compact(&st->line_buf);
+    return TF_OK;
+}
+
+static int jsonl_flush(tf_decoder *self, tf_batch ***out, size_t *n_out, tf_side_channels *side) {
+    jsonl_decoder_state *st = self->state;
+    *out = NULL;
+    *n_out = 0;
+    size_t out_cap = 0;
+
+    /* Process remaining data */
+    size_t remaining = tf_buffer_readable(&st->line_buf);
+    if (remaining > 0) {
+        uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+        size_t line_no = ++st->line_number;
+        size_t record_offset = st->byte_offset;
+        if (check_jsonl_buffer_record_limit(st, remaining, line_no, record_offset, side) != TF_OK)
+            return TF_ERROR;
+        if (process_jsonl_line(st, (const char *)buf, remaining, line_no,
+                               out, n_out, &out_cap, side) != TF_OK)
+            return TF_ERROR;
+        st->line_buf.read_pos = st->line_buf.len;
+        st->byte_offset += remaining;
+    }
+
+    /* Emit remaining batch */
+    if (st->batch && st->rows_buffered > 0) {
+        if (emit_jsonl_batch(st->batch, out, n_out, &out_cap) != TF_OK) return TF_ERROR;
+        st->batch = NULL;
+        st->rows_buffered = 0;
+    }
+
+    return TF_OK;
+}
+
+static void jsonl_decoder_destroy(tf_decoder *self) {
+    jsonl_decoder_state *st = self->state;
+    if (st) {
+        tf_buffer_free(&st->line_buf);
+        if (st->batch) tf_batch_free(st->batch);
+        if (st->col_names) {
+            for (size_t i = 0; i < st->n_cols; i++) free(st->col_names[i]);
+        }
+        free(st->col_names);
+        free(st->col_types);
+        free(st->fields);
+        free(st->aliases);
+        free(st->values);
+        free(st);
+    }
+    free(self);
+}
+
+tf_decoder *tf_jsonl_decoder_create(const cJSON *args) {
+    jsonl_decoder_state *st = calloc(1, sizeof(jsonl_decoder_state));
+    if (!st) return NULL;
+
+    st->batch_size = DEFAULT_BATCH_SIZE;
+    st->on_error = JSONL_ERROR_SKIP;
+    st->max_error_bytes = DEFAULT_MAX_ERROR_BYTES;
+    st->max_record_bytes = DEFAULT_MAX_RECORD_BYTES;
+    if (args) {
+        size_t parsed_size = 0;
+        int has_batch_size = tf_json_get_size_arg(args, "batch_size", 1, TF_MAX_BATCH_ROWS, &parsed_size, "jsonl");
+        if (has_batch_size < 0) { free(st); return NULL; }
+        if (has_batch_size > 0) st->batch_size = parsed_size;
+        cJSON *on_error = cJSON_GetObjectItemCaseSensitive(args, "on_error");
+        if (cJSON_IsString(on_error)) {
+            if (!parse_jsonl_error_action(on_error->valuestring, &st->on_error)) {
+                tf_set_last_error("jsonl: on_error must be skip, fail, warn, or quarantine");
+                free(st);
+                return NULL;
+            }
+        }
+        int has_max_error = tf_json_get_size_arg(args, "max_error_bytes", 0, TF_MAX_ERROR_BYTES, &parsed_size, "jsonl");
+        if (has_max_error < 0) { free(st); return NULL; }
+        if (has_max_error > 0) st->max_error_bytes = parsed_size;
+        int has_max_record = tf_json_get_size_arg(args, "max_record_bytes", 0, TF_MAX_RECORD_BYTES, &parsed_size, "jsonl");
+        if (has_max_record < 0) { free(st); return NULL; }
+        if (has_max_record > 0) st->max_record_bytes = parsed_size;
+    }
+
+    tf_buffer_init(&st->line_buf);
+
+    tf_decoder *dec = malloc(sizeof(tf_decoder));
+    if (!dec) { free(st); return NULL; }
+    dec->decode = jsonl_decode;
+    dec->flush = jsonl_flush;
+    dec->destroy = jsonl_decoder_destroy;
+    dec->state = st;
+    return dec;
+}
+
+/* ================================================================
+ * JSONL Encoder
+ * ================================================================ */
+
+typedef struct {
+    char  **names;
+    char  **key_prefixes;
+    size_t *key_prefix_lens;
+    size_t  n_cols;
+} jsonl_encoder_state;
+
+static int jsonl_write_hex_escape(tf_buffer *out, unsigned char ch) {
+    static const char hex[] = "0123456789abcdef";
+    char esc[6] = {
+        '\\', 'u', '0', '0',
+        hex[(ch >> 4) & 0x0f],
+        hex[ch & 0x0f]
+    };
+    return tf_buffer_write(out, (const uint8_t *)esc, sizeof(esc));
+}
+
+static int jsonl_write_escaped_string(tf_buffer *out, const char *s) {
+    if (tf_buffer_write(out, (const uint8_t *)"\"", 1) != TF_OK) return TF_ERROR;
+    const unsigned char *start = (const unsigned char *)(s ? s : "");
+    const unsigned char *p = start;
+    while (*p) {
+        unsigned char ch = *p;
+        int needs_escape = ch < 0x20 || ch == '"' || ch == '\\';
+        if (!needs_escape) {
+            p++;
+            continue;
+        }
+
+        if (p > start && tf_buffer_write(out, start, (size_t)(p - start)) != TF_OK)
+            return TF_ERROR;
+
+        int rc = TF_OK;
+        switch (ch) {
+            case '"':  rc = tf_buffer_write_str(out, "\\\""); break;
+            case '\\': rc = tf_buffer_write_str(out, "\\\\"); break;
+            case '\b': rc = tf_buffer_write_str(out, "\\b"); break;
+            case '\f': rc = tf_buffer_write_str(out, "\\f"); break;
+            case '\n': rc = tf_buffer_write_str(out, "\\n"); break;
+            case '\r': rc = tf_buffer_write_str(out, "\\r"); break;
+            case '\t': rc = tf_buffer_write_str(out, "\\t"); break;
+            default:   rc = jsonl_write_hex_escape(out, ch); break;
+        }
+        if (rc != TF_OK) return TF_ERROR;
+        p++;
+        start = p;
+    }
+    if (p > start && tf_buffer_write(out, start, (size_t)(p - start)) != TF_OK)
+        return TF_ERROR;
+    return tf_buffer_write(out, (const uint8_t *)"\"", 1);
+}
+
+static void jsonl_encoder_clear_cache(jsonl_encoder_state *st) {
+    if (!st) return;
+    if (st->names) {
+        for (size_t i = 0; i < st->n_cols; i++) free(st->names[i]);
+    }
+    if (st->key_prefixes) {
+        for (size_t i = 0; i < st->n_cols; i++) free(st->key_prefixes[i]);
+    }
+    free(st->names);
+    free(st->key_prefixes);
+    free(st->key_prefix_lens);
+    st->names = NULL;
+    st->key_prefixes = NULL;
+    st->key_prefix_lens = NULL;
+    st->n_cols = 0;
+}
+
+static int jsonl_encoder_cache_matches(const jsonl_encoder_state *st, const tf_batch *in) {
+    if (!st || !in || st->n_cols != in->n_cols || !st->names) return 0;
+    for (size_t c = 0; c < in->n_cols; c++) {
+        const char *name = in->col_names[c] ? in->col_names[c] : "";
+        if (strcmp(st->names[c], name) != 0) return 0;
+    }
+    return 1;
+}
+
+static int jsonl_build_key_prefix(const char *name, int comma,
+                                  char **out_prefix, size_t *out_len) {
+    tf_buffer tmp;
+    tf_buffer_init(&tmp);
+    int rc = TF_ERROR;
+    if (comma && tf_buffer_write(&tmp, (const uint8_t *)",", 1) != TF_OK) goto done;
+    if (jsonl_write_escaped_string(&tmp, name) != TF_OK) goto done;
+    if (tf_buffer_write(&tmp, (const uint8_t *)":", 1) != TF_OK) goto done;
+
+    size_t len = tf_buffer_readable(&tmp);
+    size_t bytes = 0;
+    if (tf_size_add(len, 1, &bytes) != TF_OK) goto done;
+    char *copy = tf_mallocarray_checked(bytes, sizeof(char));
+    if (!copy) goto done;
+    if (len > 0) memcpy(copy, tmp.data + tmp.read_pos, len);
+    copy[len] = '\0';
+    *out_prefix = copy;
+    *out_len = len;
+    rc = TF_OK;
+done:
+    tf_buffer_free(&tmp);
+    return rc;
+}
+
+static int jsonl_encoder_build_cache(jsonl_encoder_state *st, const tf_batch *in) {
+    jsonl_encoder_clear_cache(st);
+    size_t n = in->n_cols;
+    st->names = tf_callocarray_checked(n ? n : 1, sizeof(char *));
+    st->key_prefixes = tf_callocarray_checked(n ? n : 1, sizeof(char *));
+    st->key_prefix_lens = tf_callocarray_checked(n ? n : 1, sizeof(size_t));
+    if (!st->names || !st->key_prefixes || !st->key_prefix_lens) {
+        jsonl_encoder_clear_cache(st);
+        return TF_ERROR;
+    }
+    st->n_cols = n;
+
+    for (size_t c = 0; c < n; c++) {
+        const char *name = in->col_names[c] ? in->col_names[c] : "";
+        st->names[c] = tf_strdup_checked(name);
+        if (!st->names[c]) {
+            jsonl_encoder_clear_cache(st);
+            return TF_ERROR;
+        }
+        if (jsonl_build_key_prefix(name, c > 0, &st->key_prefixes[c],
+                                   &st->key_prefix_lens[c]) != TF_OK) {
+            jsonl_encoder_clear_cache(st);
+            return TF_ERROR;
+        }
+    }
+    return TF_OK;
+}
+
+static int jsonl_encoder_ensure_cache(jsonl_encoder_state *st, const tf_batch *in) {
+    if (jsonl_encoder_cache_matches(st, in)) return TF_OK;
+    return jsonl_encoder_build_cache(st, in);
+}
+
+static int jsonl_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
+    jsonl_encoder_state *st = self->state;
+    if (jsonl_encoder_ensure_cache(st, in) != TF_OK) return TF_ERROR;
+    char numbuf[64];
+#define JSONL_WRITE(expr) do { if ((expr) != TF_OK) return TF_ERROR; } while (0)
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        JSONL_WRITE(tf_buffer_write(out, (const uint8_t *)"{", 1));
+        for (size_t c = 0; c < in->n_cols; c++) {
+            JSONL_WRITE(tf_buffer_write(out, (const uint8_t *)st->key_prefixes[c],
+                                        st->key_prefix_lens[c]));
+
+            /* Value */
+            if (tf_batch_is_null(in, r, c)) {
+                JSONL_WRITE(tf_buffer_write_str(out, "null"));
+                continue;
+            }
+            switch (in->col_types[c]) {
+                case TF_TYPE_BOOL:
+                    JSONL_WRITE(tf_buffer_write_str(out, tf_batch_get_bool(in, r, c) ? "true" : "false"));
+                    break;
+                case TF_TYPE_INT64:
+                    snprintf(numbuf, sizeof(numbuf), "%lld",
+                             (long long)tf_batch_get_int64(in, r, c));
+                    JSONL_WRITE(tf_buffer_write_str(out, numbuf));
+                    break;
+                case TF_TYPE_FLOAT64:
+                    if (!isfinite(tf_batch_get_float64(in, r, c))) {
+                        tf_set_last_error("jsonl cannot encode nonfinite numbers");
+                        return TF_ERROR;
+                    }
+                    if (tf_format_float64(numbuf, sizeof(numbuf),
+                                          tf_batch_get_float64(in, r, c)) != TF_OK)
+                        return TF_ERROR;
+                    JSONL_WRITE(tf_buffer_write_str(out, numbuf));
+                    break;
+                case TF_TYPE_STRING:
+                    JSONL_WRITE(jsonl_write_escaped_string(out, tf_batch_get_string(in, r, c)));
+                    break;
+                case TF_TYPE_DATE: {
+                    char dbuf[32];
+                    tf_date_format(tf_batch_get_date(in, r, c), dbuf, sizeof(dbuf));
+                    JSONL_WRITE(jsonl_write_escaped_string(out, dbuf));
+                    break;
+                }
+                case TF_TYPE_TIMESTAMP: {
+                    char tsbuf[40];
+                    tf_timestamp_format(tf_batch_get_timestamp(in, r, c), tsbuf, sizeof(tsbuf));
+                    JSONL_WRITE(jsonl_write_escaped_string(out, tsbuf));
+                    break;
+                }
+                default:
+                    JSONL_WRITE(tf_buffer_write_str(out, "null"));
+                    break;
+            }
+        }
+        JSONL_WRITE(tf_buffer_write(out, (const uint8_t *)"}\n", 2));
+    }
+#undef JSONL_WRITE
+    return TF_OK;
+}
+
+static int jsonl_encoder_flush(tf_encoder *self, tf_buffer *out) {
+    (void)self; (void)out;
+    return TF_OK;
+}
+
+static void jsonl_encoder_destroy(tf_encoder *self) {
+    jsonl_encoder_state *st = self->state;
+    if (st) {
+        jsonl_encoder_clear_cache(st);
+        free(st);
+    }
+    free(self);
+}
+
+tf_encoder *tf_jsonl_encoder_create(const cJSON *args) {
+    (void)args;
+    jsonl_encoder_state *st = calloc(1, sizeof(jsonl_encoder_state));
+    if (!st) return NULL;
+    tf_encoder *enc = malloc(sizeof(tf_encoder));
+    if (!enc) {
+        free(st);
+        return NULL;
+    }
+    enc->encode = jsonl_encode;
+    enc->flush = jsonl_encoder_flush;
+    enc->destroy = jsonl_encoder_destroy;
+    enc->state = st;
+    return enc;
+}