tranfi 0.1.2 → 0.2.0

package/csrc/codec_text.c

@@ -15,6 +15,8 @@
 #include <string.h>
 
 #define DEFAULT_BATCH_SIZE 1024
+#define DEFAULT_MAX_ERROR_BYTES 4096
+#define DEFAULT_MAX_RECORD_BYTES (64 * 1024 * 1024)
 
 /* ================================================================
  * Text Decoder
@@ -22,6 +24,10 @@
 
 typedef struct {
     size_t    batch_size;
+    size_t    max_error_bytes;
+    size_t    max_record_bytes;
+    size_t    line_number;
+    size_t    byte_offset;
     tf_buffer line_buf;
     tf_batch *batch;
     size_t    rows_buffered;
@@ -30,70 +36,262 @@ typedef struct {
 static tf_batch *make_text_batch(size_t capacity) {
     tf_batch *b = tf_batch_create(1, capacity);
     if (!b) return NULL;
-    tf_batch_set_schema(b, 0, "_line", TF_TYPE_STRING);
+    if (tf_batch_set_schema(b, 0, "_line", TF_TYPE_STRING) != TF_OK) {
+        tf_batch_free(b);
+        return NULL;
+    }
     return b;
 }
 
+static char *text_record_preview(text_decoder_state *st,
+                                 const uint8_t *prefix, size_t prefix_len,
+                                 const uint8_t *suffix, size_t suffix_len,
+                                 int *truncated_out) {
+    size_t total = prefix_len + suffix_len;
+    size_t keep = total;
+    int truncated = 0;
+    if (st->max_error_bytes > 0 && keep > st->max_error_bytes) {
+        keep = st->max_error_bytes;
+        truncated = 1;
+    }
+    char *raw = malloc(keep + 1);
+    if (!raw) return NULL;
+    size_t copied = 0;
+    size_t n = prefix_len < keep ? prefix_len : keep;
+    if (n > 0 && prefix) {
+        memcpy(raw, prefix, n);
+        copied += n;
+    }
+    if (copied < keep && suffix) {
+        size_t m = keep - copied;
+        if (m > suffix_len) m = suffix_len;
+        if (m > 0) {
+            memcpy(raw + copied, suffix, m);
+            copied += m;
+        }
+    }
+    raw[copied] = '\0';
+    if (truncated_out) *truncated_out = truncated;
+    return raw;
+}
+
+static int emit_text_record_size_diagnostic(text_decoder_state *st,
+                                            const uint8_t *prefix, size_t prefix_len,
+                                            const uint8_t *suffix, size_t suffix_len,
+                                            size_t line_no, size_t byte_offset,
+                                            size_t observed_len,
+                                            tf_side_channels *side) {
+    if (!side || !side->errors) return TF_OK;
+    int truncated = 0;
+    char *raw = text_record_preview(st, prefix, prefix_len, suffix, suffix_len, &truncated);
+    if (!raw) return TF_ERROR;
+
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) { free(raw); return TF_ERROR; }
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "text_record_too_large") != TF_OK ||
+        tf_json_add_string(obj, "op", "codec.text.decode") != TF_OK ||
+        tf_json_add_string(obj, "action", "fail") != TF_OK ||
+        tf_json_add_string(obj, "severity", "error") != TF_OK ||
+        tf_json_add_number(obj, "line", (double)line_no) != TF_OK ||
+        tf_json_add_number(obj, "byte_offset", (double)byte_offset) != TF_OK ||
+        tf_json_add_number(obj, "max_record_bytes", (double)st->max_record_bytes) != TF_OK ||
+        tf_json_add_number(obj, "observed_bytes", (double)observed_len) != TF_OK ||
+        tf_json_add_string(obj, "message", "Text record exceeds max_record_bytes") != TF_OK ||
+        tf_json_add_number(obj, "raw_bytes", (double)observed_len) != TF_OK ||
+        tf_json_add_string(obj, "raw", raw) != TF_OK ||
+        (truncated && tf_json_add_bool(obj, "truncated", 1) != TF_OK)) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->errors, obj);
+done:
+    cJSON_Delete(obj);
+    free(raw);
+    return rc;
+}
+
+static int fail_text_record_limit(text_decoder_state *st,
+                                  const uint8_t *prefix, size_t prefix_len,
+                                  const uint8_t *suffix, size_t suffix_len,
+                                  size_t line_no, size_t byte_offset,
+                                  size_t observed_len,
+                                  tf_side_channels *side) {
+    if (emit_text_record_size_diagnostic(st, prefix, prefix_len, suffix, suffix_len,
+                                         line_no, byte_offset, observed_len, side) != TF_OK)
+        return TF_ERROR;
+    char err[256];
+    snprintf(err, sizeof(err),
+             "text record exceeds max_record_bytes at line %zu: max %zu bytes, observed %zu bytes",
+             line_no, st->max_record_bytes, observed_len);
+    tf_set_last_error(err);
+    return TF_ERROR;
+}
+
+static int check_text_buffer_record_limit(text_decoder_state *st, size_t observed_len,
+                                          size_t line_no, size_t byte_offset,
+                                          tf_side_channels *side) {
+    if (st->max_record_bytes == 0 || observed_len <= st->max_record_bytes) return TF_OK;
+    const uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+    return fail_text_record_limit(st, buf, observed_len, NULL, 0,
+                                  line_no, byte_offset, observed_len, side);
+}
+
+static int text_set_line_cell(tf_batch *batch, size_t row,
+                              const uint8_t *data, size_t len) {
+    return tf_batch_set_string_len(batch, row, 0, (const char *)data, len);
+}
+
+static int text_append_output_batch(tf_batch ***out, size_t *n_out,
+                                    size_t *out_cap, tf_batch *batch) {
+    if (*n_out >= *out_cap) {
+        size_t need = 0;
+        size_t new_cap = 0;
+        if (tf_size_add(*n_out, 1, &need) != TF_OK ||
+            tf_size_grow_pow2(*out_cap, need, 1, &new_cap) != TF_OK) {
+            return TF_ERROR;
+        }
+        tf_batch **new_out = tf_reallocarray_checked(*out, new_cap, sizeof(tf_batch *));
+        if (!new_out) return TF_ERROR;
+        *out = new_out;
+        *out_cap = new_cap;
+    }
+    (*out)[(*n_out)++] = batch;
+    return TF_OK;
+}
+
+static int text_check_segment_record_limit(text_decoder_state *st,
+                                           const uint8_t *segment, size_t segment_len,
+                                           tf_side_channels *side) {
+    if (st->max_record_bytes == 0) return TF_OK;
+
+    size_t prefix_len = tf_buffer_readable(&st->line_buf);
+    size_t total = 0;
+    if (tf_size_add(prefix_len, segment_len, &total) == TF_OK &&
+        total <= st->max_record_bytes) {
+        return TF_OK;
+    }
+
+    size_t suffix_len = segment_len;
+    size_t observed_len = total;
+    if (prefix_len >= st->max_record_bytes) {
+        suffix_len = segment_len > 0 ? 1 : 0;
+        if (tf_size_add(prefix_len, suffix_len, &observed_len) != TF_OK)
+            observed_len = st->max_record_bytes + 1;
+    } else {
+        size_t allowed = st->max_record_bytes - prefix_len;
+        suffix_len = allowed + 1;
+        if (suffix_len > segment_len) suffix_len = segment_len;
+        observed_len = prefix_len + suffix_len;
+    }
+
+    const uint8_t *prefix = prefix_len > 0
+        ? st->line_buf.data + st->line_buf.read_pos
+        : NULL;
+    return fail_text_record_limit(st, prefix, prefix_len, segment, suffix_len,
+                                  st->line_number + 1, st->byte_offset,
+                                  observed_len, side);
+}
+
+static int text_emit_line(text_decoder_state *st, const uint8_t *data, size_t len,
+                          tf_batch ***out, size_t *n_out, size_t *out_cap) {
+    if (!st->batch) {
+        st->batch = make_text_batch(st->batch_size);
+        if (!st->batch) return TF_ERROR;
+    }
+
+    size_t row = st->batch->n_rows;
+    size_t need_rows = 0;
+    if (tf_size_add(row, 1, &need_rows) != TF_OK ||
+        tf_batch_ensure_capacity(st->batch, need_rows) != TF_OK)
+        return TF_ERROR;
+
+    if (text_set_line_cell(st->batch, row, data, len) != TF_OK)
+        return TF_ERROR;
+    if (tf_batch_expose_row(st->batch, row) != TF_OK)
+        return TF_ERROR;
+    st->rows_buffered++;
+
+    if (st->rows_buffered >= st->batch_size) {
+        if (text_append_output_batch(out, n_out, out_cap, st->batch) != TF_OK)
+            return TF_ERROR;
+        st->batch = NULL;
+        st->rows_buffered = 0;
+    }
+    return TF_OK;
+}
+
+static int text_emit_complete_segment(text_decoder_state *st,
+                                      const uint8_t *segment, size_t segment_len,
+                                      tf_batch ***out, size_t *n_out,
+                                      size_t *out_cap) {
+    size_t prefix_len = tf_buffer_readable(&st->line_buf);
+    const uint8_t *line = segment;
+    size_t line_len = segment_len;
+
+    if (prefix_len > 0) {
+        if (segment_len > 0 &&
+            tf_buffer_write(&st->line_buf, segment, segment_len) != TF_OK)
+            return TF_ERROR;
+        line = st->line_buf.data + st->line_buf.read_pos;
+        line_len = tf_buffer_readable(&st->line_buf);
+    }
+
+    if (line_len > 0 && line[line_len - 1] == '\r')
+        line_len--;
+
+    if (text_emit_line(st, line, line_len, out, n_out, out_cap) != TF_OK)
+        return TF_ERROR;
+
+    if (prefix_len > 0) {
+        st->line_buf.read_pos = 0;
+        st->line_buf.len = 0;
+    }
+    return TF_OK;
+}
+
 static int text_decode(tf_decoder *self, const uint8_t *data, size_t len,
-                       tf_batch ***out, size_t *n_out) {
+                       tf_batch ***out, size_t *n_out, tf_side_channels *side) {
     text_decoder_state *st = self->state;
     *out = NULL;
     *n_out = 0;
 
-    if (tf_buffer_write(&st->line_buf, data, len) != TF_OK) return TF_ERROR;
-
     size_t out_cap = 0;
-    uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
-    size_t buf_len = st->line_buf.len - st->line_buf.read_pos;
-
-    size_t line_start = 0;
-    for (size_t i = 0; i < buf_len; i++) {
-        if (buf[i] == '\n') {
-            size_t line_len = i - line_start;
-            /* Strip trailing \r for CRLF */
-            if (line_len > 0 && buf[line_start + line_len - 1] == '\r')
-                line_len--;
-
-            if (!st->batch) {
-                st->batch = make_text_batch(st->batch_size);
-                if (!st->batch) return TF_ERROR;
-            }
+    size_t pos = 0;
 
-            size_t row = st->batch->n_rows;
-            if (tf_batch_ensure_capacity(st->batch, row + 1) != TF_OK)
-                return TF_ERROR;
+    while (pos < len) {
+        const uint8_t *start = data + pos;
+        const uint8_t *nl = memchr(start, '\n', len - pos);
+        size_t segment_len = nl ? (size_t)(nl - start) : len - pos;
 
-            /* Copy line into batch */
-            char *line_str = malloc(line_len + 1);
-            if (!line_str) return TF_ERROR;
-            memcpy(line_str, buf + line_start, line_len);
-            line_str[line_len] = '\0';
-            tf_batch_set_string(st->batch, row, 0, line_str);
-            free(line_str);
-            st->batch->n_rows = row + 1;
-            st->rows_buffered++;
-
-            line_start = i + 1;
-
-            /* Emit batch if full */
-            if (st->rows_buffered >= st->batch_size) {
-                if (*n_out >= out_cap) {
-                    out_cap = (out_cap == 0) ? 4 : out_cap * 2;
-                    *out = realloc(*out, out_cap * sizeof(tf_batch *));
-                }
-                (*out)[(*n_out)++] = st->batch;
-                st->batch = NULL;
-                st->rows_buffered = 0;
-            }
+        if (text_check_segment_record_limit(st, start, segment_len, side) != TF_OK)
+            return TF_ERROR;
+
+        if (nl) {
+            size_t prefix_len = tf_buffer_readable(&st->line_buf);
+            if (text_emit_complete_segment(st, start, segment_len,
+                                           out, n_out, &out_cap) != TF_OK)
+                return TF_ERROR;
+            size_t advance = 0;
+            size_t new_offset = 0;
+            if (tf_size_add(prefix_len, segment_len, &advance) != TF_OK ||
+                tf_size_add(advance, 1, &advance) != TF_OK ||
+                tf_size_add(st->byte_offset, advance, &new_offset) != TF_OK)
+                return TF_ERROR;
+            st->line_number++;
+            st->byte_offset = new_offset;
+            pos += segment_len + 1;
+        } else {
+            if (segment_len > 0 &&
+                tf_buffer_write(&st->line_buf, start, segment_len) != TF_OK)
+                return TF_ERROR;
+            pos = len;
         }
     }
-
-    st->line_buf.read_pos += line_start;
-    tf_buffer_compact(&st->line_buf);
     return TF_OK;
 }
 
-static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out) {
+static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out, tf_side_channels *side) {
+    (void)side;
     text_decoder_state *st = self->state;
     *out = NULL;
     *n_out = 0;
@@ -103,6 +301,11 @@ static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out) {
     size_t remaining = tf_buffer_readable(&st->line_buf);
     if (remaining > 0) {
         uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+        size_t line_no = st->line_number + 1;
+        size_t record_offset = st->byte_offset;
+        if (check_text_buffer_record_limit(st, remaining, line_no, record_offset, side) != TF_OK)
+            return TF_ERROR;
+        st->line_number = line_no;
 
         if (!st->batch) {
             st->batch = make_text_batch(st->batch_size);
@@ -110,7 +313,9 @@ static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out) {
         }
 
         size_t row = st->batch->n_rows;
-        if (tf_batch_ensure_capacity(st->batch, row + 1) != TF_OK)
+        size_t need_rows = 0;
+        if (tf_size_add(row, 1, &need_rows) != TF_OK ||
+            tf_batch_ensure_capacity(st->batch, need_rows) != TF_OK)
             return TF_ERROR;
 
         /* Strip trailing \r */
@@ -118,25 +323,23 @@ static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out) {
         if (line_len > 0 && buf[line_len - 1] == '\r')
             line_len--;
 
-        char *line_str = malloc(line_len + 1);
-        if (!line_str) return TF_ERROR;
-        memcpy(line_str, buf, line_len);
-        line_str[line_len] = '\0';
-        tf_batch_set_string(st->batch, row, 0, line_str);
-        free(line_str);
-        st->batch->n_rows = row + 1;
+        if (text_set_line_cell(st->batch, row, buf, line_len) != TF_OK)
+            return TF_ERROR;
+        if (tf_batch_expose_row(st->batch, row) != TF_OK)
+            return TF_ERROR;
         st->rows_buffered++;
 
         st->line_buf.read_pos = st->line_buf.len;
+        size_t new_offset = 0;
+        if (tf_size_add(st->byte_offset, remaining, &new_offset) != TF_OK)
+            return TF_ERROR;
+        st->byte_offset = new_offset;
     }
 
     /* Emit remaining batch */
     if (st->batch && st->rows_buffered > 0) {
-        if (*n_out >= out_cap) {
-            out_cap = (*n_out == 0) ? 1 : out_cap * 2;
-            *out = realloc(*out, out_cap * sizeof(tf_batch *));
-        }
-        (*out)[(*n_out)++] = st->batch;
+        if (text_append_output_batch(out, n_out, &out_cap, st->batch) != TF_OK)
+            return TF_ERROR;
         st->batch = NULL;
         st->rows_buffered = 0;
     }
@@ -159,10 +362,19 @@ tf_decoder *tf_text_decoder_create(const cJSON *args) {
     if (!st) return NULL;
 
     st->batch_size = DEFAULT_BATCH_SIZE;
+    st->max_error_bytes = DEFAULT_MAX_ERROR_BYTES;
+    st->max_record_bytes = DEFAULT_MAX_RECORD_BYTES;
     if (args) {
-        cJSON *bs = cJSON_GetObjectItemCaseSensitive(args, "batch_size");
-        if (cJSON_IsNumber(bs) && bs->valueint > 0)
-            st->batch_size = (size_t)bs->valueint;
+        size_t parsed_size = 0;
+        int has_batch_size = tf_json_get_size_arg(args, "batch_size", 1, TF_MAX_BATCH_ROWS, &parsed_size, "text");
+        if (has_batch_size < 0) { free(st); return NULL; }
+        if (has_batch_size > 0) st->batch_size = parsed_size;
+        int has_max_error = tf_json_get_size_arg(args, "max_error_bytes", 0, TF_MAX_ERROR_BYTES, &parsed_size, "text");
+        if (has_max_error < 0) { free(st); return NULL; }
+        if (has_max_error > 0) st->max_error_bytes = parsed_size;
+        int has_max_record = tf_json_get_size_arg(args, "max_record_bytes", 0, TF_MAX_RECORD_BYTES, &parsed_size, "text");
+        if (has_max_record < 0) { free(st); return NULL; }
+        if (has_max_record > 0) st->max_record_bytes = parsed_size;
     }
 
     tf_buffer_init(&st->line_buf);
@@ -180,6 +392,11 @@ tf_decoder *tf_text_decoder_create(const cJSON *args) {
  * Text Encoder
  * ================================================================ */
 
+static int text_write_string(tf_buffer *out, const char *s) {
+    const char *value = s ? s : "";
+    return tf_buffer_write(out, (const uint8_t *)value, strlen(value));
+}
+
 static int text_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
     (void)self;
 
@@ -191,19 +408,20 @@ static int text_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
             /* Write _line column value */
             if (!tf_batch_is_null(in, r, (size_t)line_col)) {
                 const char *s = tf_batch_get_string(in, r, (size_t)line_col);
-                tf_buffer_write(out, (const uint8_t *)s, strlen(s));
+                if (text_write_string(out, s) != TF_OK) return TF_ERROR;
             }
         } else {
             /* Fallback: concatenate all string columns with tab */
             for (size_t c = 0; c < in->n_cols; c++) {
-                if (c > 0) tf_buffer_write(out, (const uint8_t *)"\t", 1);
+                if (c > 0 && tf_buffer_write(out, (const uint8_t *)"\t", 1) != TF_OK)
+                    return TF_ERROR;
                 if (!tf_batch_is_null(in, r, c) && in->col_types[c] == TF_TYPE_STRING) {
                     const char *s = tf_batch_get_string(in, r, c);
-                    tf_buffer_write(out, (const uint8_t *)s, strlen(s));
+                    if (text_write_string(out, s) != TF_OK) return TF_ERROR;
                 }
             }
         }
-        tf_buffer_write(out, (const uint8_t *)"\n", 1);
+        if (tf_buffer_write(out, (const uint8_t *)"\n", 1) != TF_OK) return TF_ERROR;
     }
     return TF_OK;
 }

package/csrc/compiler.c

@@ -3,7 +3,7 @@
  *
  * Iterates IR nodes, looks up each op in the registry,
  * and calls create_native() to build live decoder/steps/encoder structs.
- * Replaces the constructor dispatch loop that was in plan.c.
+ * This is the single native construction path for JSON IR plans.
  */
 
 #include "ir.h"
@@ -31,6 +31,11 @@ int tf_compile_native(const tf_ir_plan *plan,
     *out_encoder = NULL;
     if (error) *error = NULL;
 
+    if (!plan || plan->n_nodes == 0) {
+        set_error(error, "empty plan");
+        return TF_ERROR;
+    }
+
     /* Allocate steps array (max = n_nodes, since some are decoder/encoder) */
     tf_step **steps = calloc(plan->n_nodes, sizeof(tf_step *));
     if (!steps) {
@@ -66,17 +71,31 @@ int tf_compile_native(const tf_ir_plan *plan,
 
         void *obj = entry->create_native(node->args);
         if (!obj) {
-            char buf[256];
-            snprintf(buf, sizeof(buf), "failed to create '%s'", node->op);
+            const char *detail = tf_last_error();
+            char buf[512];
+            if (detail && detail[0])
+                snprintf(buf, sizeof(buf), "failed to create '%s': %s", node->op, detail);
+            else
+                snprintf(buf, sizeof(buf), "failed to create '%s'", node->op);
             set_error(error, buf);
             goto fail;
         }
 
         switch (entry->kind) {
         case TF_OP_DECODER:
+            if (decoder) {
+                ((tf_decoder *)obj)->destroy((tf_decoder *)obj);
+                set_error(error, "multiple decoders");
+                goto fail;
+            }
             decoder = (tf_decoder *)obj;
             break;
         case TF_OP_ENCODER:
+            if (encoder) {
+                ((tf_encoder *)obj)->destroy((tf_encoder *)obj);
+                set_error(error, "multiple encoders");
+                goto fail;
+            }
             encoder = (tf_encoder *)obj;
             break;
         case TF_OP_TRANSFORM:
@@ -85,6 +104,15 @@ int tf_compile_native(const tf_ir_plan *plan,
         }
     }
 
+    if (!decoder) {
+        set_error(error, "plan missing decoder");
+        goto fail;
+    }
+    if (!encoder) {
+        set_error(error, "plan missing encoder");
+        goto fail;
+    }
+
     *out_decoder = decoder;
     *out_steps = steps;
     *out_n_steps = n_steps;

package/csrc/config.h

@@ -0,0 +1,21 @@
+/*
+ * config.h -- Tranfi C feature-test configuration.
+ *
+ * Build files also define these macros at compile-command level so they are
+ * visible before any system header in every translation unit. This header
+ * documents the source contract and protects internal/public headers included
+ * first.
+ */
+
+#ifndef TF_CONFIG_H
+#define TF_CONFIG_H
+
+#ifndef _POSIX_C_SOURCE
+#define _POSIX_C_SOURCE 200809L
+#endif
+
+#ifndef _XOPEN_SOURCE
+#define _XOPEN_SOURCE 700
+#endif
+
+#endif /* TF_CONFIG_H */