tranfi 0.1.2 → 0.2.0

package/csrc/op_fill_down.c

@@ -27,6 +27,22 @@ typedef struct {
     int      initialized;
 } fill_down_state;
 
+static int fill_down_is_target(const fill_down_state *st, const tf_batch *in, size_t c) {
+    if (st->n_cols == 0) return 1;
+    for (size_t k = 0; k < st->n_cols; k++) {
+        if (strcmp(in->col_names[c], st->cols[k]) == 0) return 1;
+    }
+    return 0;
+}
+
+static void fill_down_free_string_slots(char **slots, size_t n) {
+    if (!slots) return;
+    for (size_t i = 0; i < n; i++) {
+        free(slots[i]);
+        slots[i] = NULL;
+    }
+}
+
 static int fill_down_process(tf_step *self, tf_batch *in, tf_batch **out,
                              tf_side_channels *side) {
     (void)side;
@@ -34,93 +50,136 @@ static int fill_down_process(tf_step *self, tf_batch *in, tf_batch **out,
     *out = NULL;
 
     if (!st->initialized) {
+        char **last_vals = calloc(in->n_cols, sizeof(char *));
+        int *last_types = calloc(in->n_cols, sizeof(int));
+        int64_t *last_int = calloc(in->n_cols, sizeof(int64_t));
+        double *last_float = calloc(in->n_cols, sizeof(double));
+        int *last_bool = calloc(in->n_cols, sizeof(int));
+        int32_t *last_date = calloc(in->n_cols, sizeof(int32_t));
+        int64_t *last_timestamp = calloc(in->n_cols, sizeof(int64_t));
+        if (!last_vals || !last_types || !last_int || !last_float ||
+            !last_bool || !last_date || !last_timestamp) {
+            free(last_vals);
+            free(last_types);
+            free(last_int);
+            free(last_float);
+            free(last_bool);
+            free(last_date);
+            free(last_timestamp);
+            return TF_ERROR;
+        }
+        for (size_t c = 0; c < in->n_cols; c++) last_types[c] = -1; /* no value yet */
         st->n_tracked = (int)in->n_cols;
-        st->last_vals = calloc(in->n_cols, sizeof(char *));
-        st->last_types = calloc(in->n_cols, sizeof(int));
-        st->last_int = calloc(in->n_cols, sizeof(int64_t));
-        st->last_float = calloc(in->n_cols, sizeof(double));
-        st->last_bool = calloc(in->n_cols, sizeof(int));
-        st->last_date = calloc(in->n_cols, sizeof(int32_t));
-        st->last_timestamp = calloc(in->n_cols, sizeof(int64_t));
-        for (size_t c = 0; c < in->n_cols; c++) st->last_types[c] = -1; /* no value yet */
+        st->last_vals = last_vals;
+        st->last_types = last_types;
+        st->last_int = last_int;
+        st->last_float = last_float;
+        st->last_bool = last_bool;
+        st->last_date = last_date;
+        st->last_timestamp = last_timestamp;
         st->initialized = 1;
     }
 
     tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
     if (!ob) return TF_ERROR;
-    for (size_t c = 0; c < in->n_cols; c++)
-        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    char **pending_strings = calloc(in->n_cols, sizeof(char *));
+    if (!pending_strings) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
 
     for (size_t r = 0; r < in->n_rows; r++) {
-        tf_batch_copy_row(ob, r, in, r);
+        memset(pending_strings, 0, sizeof(char *) * in->n_cols);
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            goto fail;
+        }
 
         for (size_t c = 0; c < in->n_cols; c++) {
-            /* Check if this column is targeted */
-            int target = 0;
-            if (st->n_cols == 0) {
-                target = 1;
-            } else {
-                for (size_t k = 0; k < st->n_cols; k++) {
-                    if (strcmp(in->col_names[c], st->cols[k]) == 0) { target = 1; break; }
-                }
-            }
-            if (!target) continue;
+            if (!fill_down_is_target(st, in, c)) continue;
 
             if (!tf_batch_is_null(in, r, c)) {
-                /* Update last value */
-                st->last_types[c] = (int)in->col_types[c];
-                switch (in->col_types[c]) {
-                    case TF_TYPE_STRING:
-                        free(st->last_vals[c]);
-                        st->last_vals[c] = strdup(tf_batch_get_string(in, r, c));
-                        break;
-                    case TF_TYPE_INT64:
-                        st->last_int[c] = tf_batch_get_int64(in, r, c); break;
-                    case TF_TYPE_FLOAT64:
-                        st->last_float[c] = tf_batch_get_float64(in, r, c); break;
-                    case TF_TYPE_BOOL:
-                        st->last_bool[c] = tf_batch_get_bool(in, r, c); break;
-                    case TF_TYPE_DATE:
-                        st->last_date[c] = tf_batch_get_date(in, r, c); break;
-                    case TF_TYPE_TIMESTAMP:
-                        st->last_timestamp[c] = tf_batch_get_timestamp(in, r, c); break;
-                    default: break;
+                if (in->col_types[c] == TF_TYPE_STRING) {
+                    const char *src = tf_batch_get_string(in, r, c);
+                    pending_strings[c] = strdup(src ? src : "");
+                    if (!pending_strings[c]) {
+                        goto fail;
+                    }
                 }
             } else if (st->last_types[c] >= 0) {
                 /* Fill with last value */
+                int write_rc = TF_OK;
                 switch (in->col_types[c]) {
                     case TF_TYPE_STRING:
-                        if (st->last_vals[c]) tf_batch_set_string(ob, r, c, st->last_vals[c]);
+                        if (st->last_vals[c]) write_rc = tf_batch_set_string(ob, r, c, st->last_vals[c]);
                         break;
                     case TF_TYPE_INT64:
-                        tf_batch_set_int64(ob, r, c, st->last_int[c]); break;
+                        write_rc = tf_batch_set_int64(ob, r, c, st->last_int[c]); break;
                     case TF_TYPE_FLOAT64:
-                        tf_batch_set_float64(ob, r, c, st->last_float[c]); break;
+                        write_rc = tf_batch_set_float64(ob, r, c, st->last_float[c]); break;
                     case TF_TYPE_BOOL:
-                        tf_batch_set_bool(ob, r, c, st->last_bool[c]); break;
+                        write_rc = tf_batch_set_bool(ob, r, c, st->last_bool[c]); break;
                     case TF_TYPE_DATE:
-                        tf_batch_set_date(ob, r, c, st->last_date[c]); break;
+                        write_rc = tf_batch_set_date(ob, r, c, st->last_date[c]); break;
                     case TF_TYPE_TIMESTAMP:
-                        tf_batch_set_timestamp(ob, r, c, st->last_timestamp[c]); break;
+                        write_rc = tf_batch_set_timestamp(ob, r, c, st->last_timestamp[c]); break;
                     default: break;
                 }
+                if (write_rc != TF_OK) goto fail;
             }
         }
-        ob->n_rows = r + 1;
+
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (!fill_down_is_target(st, in, c) || tf_batch_is_null(in, r, c)) continue;
+
+            st->last_types[c] = (int)in->col_types[c];
+            switch (in->col_types[c]) {
+                case TF_TYPE_STRING:
+                    free(st->last_vals[c]);
+                    st->last_vals[c] = pending_strings[c];
+                    pending_strings[c] = NULL;
+                    break;
+                case TF_TYPE_INT64:
+                    st->last_int[c] = tf_batch_get_int64(in, r, c); break;
+                case TF_TYPE_FLOAT64:
+                    st->last_float[c] = tf_batch_get_float64(in, r, c); break;
+                case TF_TYPE_BOOL:
+                    st->last_bool[c] = tf_batch_get_bool(in, r, c); break;
+                case TF_TYPE_DATE:
+                    st->last_date[c] = tf_batch_get_date(in, r, c); break;
+                case TF_TYPE_TIMESTAMP:
+                    st->last_timestamp[c] = tf_batch_get_timestamp(in, r, c); break;
+                default:
+                    break;
+            }
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) goto fail;
     }
 
+    free(pending_strings);
     *out = ob;
     return TF_OK;
+
+fail:
+    fill_down_free_string_slots(pending_strings, in->n_cols);
+    free(pending_strings);
+    tf_batch_free(ob);
+    return TF_ERROR;
 }
 
 static int fill_down_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
     (void)self; (void)side; *out = NULL; return TF_OK;
 }
 
-static void fill_down_destroy(tf_step *self) {
-    fill_down_state *st = self->state;
+static void fill_down_state_free(fill_down_state *st) {
     if (st) {
-        for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
+        if (st->cols) {
+            for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
+        }
         free(st->cols);
         if (st->last_vals) {
             for (int i = 0; i < st->n_tracked; i++) free(st->last_vals[i]);
@@ -131,6 +190,10 @@ static void fill_down_destroy(tf_step *self) {
         free(st->last_date); free(st->last_timestamp);
         free(st);
     }
+}
+
+static void fill_down_destroy(tf_step *self) {
+    fill_down_state_free(self->state);
     free(self);
 }
 
@@ -144,17 +207,26 @@ tf_step *tf_fill_down_create(const cJSON *args) {
             int n = cJSON_GetArraySize(columns);
             if (n > 0) {
                 st->cols = calloc(n, sizeof(char *));
-                st->n_cols = n;
+                if (!st->cols) { fill_down_state_free(st); return NULL; }
+                st->n_cols = (size_t)n;
                 for (int i = 0; i < n; i++) {
                     cJSON *item = cJSON_GetArrayItem(columns, i);
-                    if (cJSON_IsString(item)) st->cols[i] = strdup(item->valuestring);
+                    if (!cJSON_IsString(item) || !item->valuestring[0]) {
+                        fill_down_state_free(st);
+                        return NULL;
+                    }
+                    st->cols[i] = strdup(item->valuestring);
+                    if (!st->cols[i]) {
+                        fill_down_state_free(st);
+                        return NULL;
+                    }
                 }
             }
         }
     }
 
-    tf_step *step = malloc(sizeof(tf_step));
-    if (!step) { free(st); return NULL; }
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { fill_down_state_free(st); return NULL; }
     step->process = fill_down_process;
     step->flush = fill_down_flush;
     step->destroy = fill_down_destroy;

package/csrc/op_fill_null.c

@@ -10,71 +10,182 @@
 #include <stdlib.h>
 #include <string.h>
 
+/* Fill-null is row-local. Audit records are opt-in and capped so diagnostics
+ * cannot become an unbounded second output stream. */
 typedef struct {
     char **col_names;
     char **defaults;
     size_t n;
+    size_t row_index;
+    size_t audit_limit;
+    size_t audit_emitted;
+    int audit;
+    tf_audit_options audit_opts;
 } fill_null_state;
 
+static int emit_fill_null_audit(fill_null_state *st, const tf_batch *b, size_t row,
+                                size_t col, size_t row_no, tf_side_channels *side) {
+    if (!st->audit || st->audit_emitted >= st->audit_limit || !side || !side->stats) return TF_OK;
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) return TF_ERROR;
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "audit") != TF_OK ||
+        tf_json_add_string(obj, "op", "fill-null") != TF_OK ||
+        tf_json_add_string(obj, "event", "null_filled") != TF_OK ||
+        tf_json_add_string(obj, "reason", "null_filled") != TF_OK ||
+        tf_json_add_string(obj, "channel", "audit") != TF_OK ||
+        tf_json_add_string(obj, "column", b->col_names[col] ? b->col_names[col] : "") != TF_OK ||
+        tf_json_add_number(obj, "row", (double)row_no) != TF_OK ||
+        tf_json_add_null(obj, "before") != TF_OK) {
+        goto done;
+    }
+    cJSON *after = tf_audit_cell_to_json(b, row, col, &st->audit_opts);
+    if (!after || tf_json_add_item(obj, "after", after) != TF_OK) goto done;
+    cJSON *row_obj = tf_audit_row_to_json(b, row, &st->audit_opts);
+    if (row_obj) {
+        if (tf_json_add_item(obj, "data", row_obj) != TF_OK) goto done;
+    } else if (st->audit_opts.include_row) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->stats, obj);
+done:
+    cJSON_Delete(obj);
+    if (rc == TF_OK) st->audit_emitted++;
+    return rc;
+}
+
 static int fill_null_process(tf_step *self, tf_batch *in, tf_batch **out,
                              tf_side_channels *side) {
-    (void)side;
     fill_null_state *st = self->state;
     *out = NULL;
+    size_t row_base = st->row_index;
 
     tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
     if (!ob) return TF_ERROR;
-    for (size_t c = 0; c < in->n_cols; c++)
-        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
-    for (size_t r = 0; r < in->n_rows; r++) {
-        tf_batch_copy_row(ob, r, in, r);
-        ob->n_rows = r + 1;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
     }
-
-    /* Fill nulls */
-    for (size_t k = 0; k < st->n; k++) {
-        int ci = tf_batch_col_index(ob, st->col_names[k]);
-        if (ci < 0) continue;
-        for (size_t r = 0; r < ob->n_rows; r++) {
-            if (!tf_batch_is_null(ob, r, ci)) continue;
+    size_t *changed_cols = tf_callocarray_checked(st->n > 0 ? st->n : 1, sizeof(size_t));
+    if (!changed_cols) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            free(changed_cols);
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        size_t changed_count = 0;
+        for (size_t k = 0; k < st->n; k++) {
+            int ci = tf_batch_col_index(in, st->col_names[k]);
+            if (ci < 0 || !ob->nulls[ci] || !ob->nulls[ci][r]) continue;
             const char *def = st->defaults[k];
-            switch (ob->col_types[ci]) {
+            int changed = 0;
+            switch (in->col_types[ci]) {
                 case TF_TYPE_STRING:
-                    tf_batch_set_string(ob, r, ci, def);
+                    if (tf_batch_set_string(ob, r, (size_t)ci, def) != TF_OK) {
+                        free(changed_cols);
+                        tf_batch_free(ob);
+                        return TF_ERROR;
+                    }
+                    changed = 1;
                     break;
                 case TF_TYPE_INT64: {
                     char *end;
                     int64_t v = strtoll(def, &end, 10);
-                    if (*end == '\0') tf_batch_set_int64(ob, r, ci, v);
+                    if (*end == '\0') {
+                        if (tf_batch_set_int64(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
                     break;
                 }
                 case TF_TYPE_FLOAT64: {
                     char *end;
                     double v = strtod(def, &end);
-                    if (*end == '\0') tf_batch_set_float64(ob, r, ci, v);
+                    if (*end == '\0') {
+                        if (tf_batch_set_float64(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
                     break;
                 }
                 case TF_TYPE_BOOL:
-                    tf_batch_set_bool(ob, r, ci, strcmp(def, "true") == 0);
+                    if (tf_batch_set_bool(ob, r, (size_t)ci, strcmp(def, "true") == 0) != TF_OK) {
+                        free(changed_cols);
+                        tf_batch_free(ob);
+                        return TF_ERROR;
+                    }
+                    changed = 1;
                     break;
                 case TF_TYPE_DATE: {
                     char *end;
                     int32_t v = (int32_t)strtol(def, &end, 10);
-                    if (*end == '\0') tf_batch_set_date(ob, r, ci, v);
+                    if (*end == '\0') {
+                        if (tf_batch_set_date(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
                     break;
                 }
                 case TF_TYPE_TIMESTAMP: {
                     char *end;
                     int64_t v = strtoll(def, &end, 10);
-                    if (*end == '\0') tf_batch_set_timestamp(ob, r, ci, v);
+                    if (*end == '\0') {
+                        if (tf_batch_set_timestamp(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
                     break;
                 }
                 default:
                     break;
             }
+            if (changed) changed_cols[changed_count++] = (size_t)ci;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            free(changed_cols);
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (changed_count > 0) {
+            size_t row_no = 0;
+            if (tf_size_add(row_base, r, &row_no) != TF_OK ||
+                tf_size_add(row_no, 1, &row_no) != TF_OK) {
+                free(changed_cols);
+                tf_batch_free(ob);
+                return TF_ERROR;
+            }
+            for (size_t i = 0; i < changed_count; i++) {
+                if (emit_fill_null_audit(st, ob, r, changed_cols[i], row_no, side) != TF_OK) {
+                    free(changed_cols);
+                    tf_batch_free(ob);
+                    return TF_ERROR;
+                }
+            }
         }
     }
 
+    if (tf_size_add(st->row_index, in->n_rows, &st->row_index) != TF_OK) {
+        free(changed_cols);
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    free(changed_cols);
     *out = ob;
     return TF_OK;
 }
@@ -83,13 +194,22 @@ static int fill_null_flush(tf_step *self, tf_batch **out, tf_side_channels *side
     (void)self; (void)side; *out = NULL; return TF_OK;
 }
 
-static void fill_null_destroy(tf_step *self) {
-    fill_null_state *st = self->state;
-    if (st) {
-        for (size_t i = 0; i < st->n; i++) { free(st->col_names[i]); free(st->defaults[i]); }
-        free(st->col_names); free(st->defaults);
-        free(st);
+static void fill_null_state_free(fill_null_state *st) {
+    if (!st) return;
+    tf_audit_options_free(&st->audit_opts);
+    if (st->col_names) {
+        for (size_t i = 0; i < st->n; i++) free(st->col_names[i]);
     }
+    if (st->defaults) {
+        for (size_t i = 0; i < st->n; i++) free(st->defaults[i]);
+    }
+    free(st->col_names);
+    free(st->defaults);
+    free(st);
+}
+
+static void fill_null_destroy(tf_step *self) {
+    if (self) fill_null_state_free(self->state);
     free(self);
 }
 
@@ -101,20 +221,45 @@ tf_step *tf_fill_null_create(const cJSON *args) {
     int n = cJSON_GetArraySize(mapping);
     fill_null_state *st = calloc(1, sizeof(fill_null_state));
     if (!st) return NULL;
-    st->col_names = calloc(n, sizeof(char *));
-    st->defaults = calloc(n, sizeof(char *));
-    st->n = n;
+    tf_audit_options_init(&st->audit_opts, 1);
+    st->col_names = tf_callocarray_checked(n > 0 ? (size_t)n : 1, sizeof(char *));
+    st->defaults = tf_callocarray_checked(n > 0 ? (size_t)n : 1, sizeof(char *));
+    st->n = (size_t)n;
+    st->audit_limit = 1000;
+    if (!st->col_names || !st->defaults) { fill_null_state_free(st); return NULL; }
 
     int i = 0;
     cJSON *entry = NULL;
     cJSON_ArrayForEach(entry, mapping) {
+        if (!entry->string || !entry->string[0]) { fill_null_state_free(st); return NULL; }
         st->col_names[i] = strdup(entry->string);
         st->defaults[i] = strdup(cJSON_IsString(entry) ? entry->valuestring : "");
+        if (!st->col_names[i] || !st->defaults[i]) { fill_null_state_free(st); return NULL; }
         i++;
     }
 
-    tf_step *step = malloc(sizeof(tf_step));
-    if (!step) { fill_null_destroy(&(tf_step){.state = st}); return NULL; }
+    cJSON *audit_j = cJSON_GetObjectItemCaseSensitive(args, "audit");
+    st->audit = cJSON_IsTrue(audit_j) ? 1 : 0;
+    cJSON *audit_limit_j = cJSON_GetObjectItemCaseSensitive(args, "audit_limit");
+    if (!audit_limit_j) audit_limit_j = cJSON_GetObjectItemCaseSensitive(args, "auditLimit");
+    if (audit_limit_j) {
+        size_t parsed_limit = 0;
+        if (tf_json_get_size_arg_any(args, "audit_limit", "auditLimit",
+                                     1, TF_MAX_AUDIT_RECORDS,
+                                     &parsed_limit, "fill-null") < 0) {
+            fill_null_state_free(st);
+            return NULL;
+        }
+        st->audit_limit = parsed_limit;
+    }
+
+    if (tf_audit_options_parse(&st->audit_opts, args, "fill-null") != TF_OK) {
+        fill_null_state_free(st);
+        return NULL;
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { fill_null_state_free(st); return NULL; }
     step->process = fill_null_process;
     step->flush = fill_null_flush;
     step->destroy = fill_null_destroy;