tranfi 0.0.2 → 0.2.0

package/csrc/op_fill_down.c

@@ -0,0 +1,235 @@
+/*
+ * op_fill_down.c — Forward-fill nulls with last non-null value.
+ *
+ * Config: {"columns": ["city", "state"]}
+ *   or {} for all columns.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include "date_utils.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char  **cols;
+    size_t  n_cols;
+    /* Last non-null value per column (stored as strings for simplicity) */
+    char  **last_vals;
+    int    *last_types; /* tf_type */
+    int64_t *last_int;
+    double  *last_float;
+    int     *last_bool;
+    int32_t *last_date;
+    int64_t *last_timestamp;
+    int      n_tracked;
+    int      initialized;
+} fill_down_state;
+
+static int fill_down_is_target(const fill_down_state *st, const tf_batch *in, size_t c) {
+    if (st->n_cols == 0) return 1;
+    for (size_t k = 0; k < st->n_cols; k++) {
+        if (strcmp(in->col_names[c], st->cols[k]) == 0) return 1;
+    }
+    return 0;
+}
+
+static void fill_down_free_string_slots(char **slots, size_t n) {
+    if (!slots) return;
+    for (size_t i = 0; i < n; i++) {
+        free(slots[i]);
+        slots[i] = NULL;
+    }
+}
+
+static int fill_down_process(tf_step *self, tf_batch *in, tf_batch **out,
+                             tf_side_channels *side) {
+    (void)side;
+    fill_down_state *st = self->state;
+    *out = NULL;
+
+    if (!st->initialized) {
+        char **last_vals = calloc(in->n_cols, sizeof(char *));
+        int *last_types = calloc(in->n_cols, sizeof(int));
+        int64_t *last_int = calloc(in->n_cols, sizeof(int64_t));
+        double *last_float = calloc(in->n_cols, sizeof(double));
+        int *last_bool = calloc(in->n_cols, sizeof(int));
+        int32_t *last_date = calloc(in->n_cols, sizeof(int32_t));
+        int64_t *last_timestamp = calloc(in->n_cols, sizeof(int64_t));
+        if (!last_vals || !last_types || !last_int || !last_float ||
+            !last_bool || !last_date || !last_timestamp) {
+            free(last_vals);
+            free(last_types);
+            free(last_int);
+            free(last_float);
+            free(last_bool);
+            free(last_date);
+            free(last_timestamp);
+            return TF_ERROR;
+        }
+        for (size_t c = 0; c < in->n_cols; c++) last_types[c] = -1; /* no value yet */
+        st->n_tracked = (int)in->n_cols;
+        st->last_vals = last_vals;
+        st->last_types = last_types;
+        st->last_int = last_int;
+        st->last_float = last_float;
+        st->last_bool = last_bool;
+        st->last_date = last_date;
+        st->last_timestamp = last_timestamp;
+        st->initialized = 1;
+    }
+
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    char **pending_strings = calloc(in->n_cols, sizeof(char *));
+    if (!pending_strings) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        memset(pending_strings, 0, sizeof(char *) * in->n_cols);
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            goto fail;
+        }
+
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (!fill_down_is_target(st, in, c)) continue;
+
+            if (!tf_batch_is_null(in, r, c)) {
+                if (in->col_types[c] == TF_TYPE_STRING) {
+                    const char *src = tf_batch_get_string(in, r, c);
+                    pending_strings[c] = strdup(src ? src : "");
+                    if (!pending_strings[c]) {
+                        goto fail;
+                    }
+                }
+            } else if (st->last_types[c] >= 0) {
+                /* Fill with last value */
+                int write_rc = TF_OK;
+                switch (in->col_types[c]) {
+                    case TF_TYPE_STRING:
+                        if (st->last_vals[c]) write_rc = tf_batch_set_string(ob, r, c, st->last_vals[c]);
+                        break;
+                    case TF_TYPE_INT64:
+                        write_rc = tf_batch_set_int64(ob, r, c, st->last_int[c]); break;
+                    case TF_TYPE_FLOAT64:
+                        write_rc = tf_batch_set_float64(ob, r, c, st->last_float[c]); break;
+                    case TF_TYPE_BOOL:
+                        write_rc = tf_batch_set_bool(ob, r, c, st->last_bool[c]); break;
+                    case TF_TYPE_DATE:
+                        write_rc = tf_batch_set_date(ob, r, c, st->last_date[c]); break;
+                    case TF_TYPE_TIMESTAMP:
+                        write_rc = tf_batch_set_timestamp(ob, r, c, st->last_timestamp[c]); break;
+                    default: break;
+                }
+                if (write_rc != TF_OK) goto fail;
+            }
+        }
+
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (!fill_down_is_target(st, in, c) || tf_batch_is_null(in, r, c)) continue;
+
+            st->last_types[c] = (int)in->col_types[c];
+            switch (in->col_types[c]) {
+                case TF_TYPE_STRING:
+                    free(st->last_vals[c]);
+                    st->last_vals[c] = pending_strings[c];
+                    pending_strings[c] = NULL;
+                    break;
+                case TF_TYPE_INT64:
+                    st->last_int[c] = tf_batch_get_int64(in, r, c); break;
+                case TF_TYPE_FLOAT64:
+                    st->last_float[c] = tf_batch_get_float64(in, r, c); break;
+                case TF_TYPE_BOOL:
+                    st->last_bool[c] = tf_batch_get_bool(in, r, c); break;
+                case TF_TYPE_DATE:
+                    st->last_date[c] = tf_batch_get_date(in, r, c); break;
+                case TF_TYPE_TIMESTAMP:
+                    st->last_timestamp[c] = tf_batch_get_timestamp(in, r, c); break;
+                default:
+                    break;
+            }
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) goto fail;
+    }
+
+    free(pending_strings);
+    *out = ob;
+    return TF_OK;
+
+fail:
+    fill_down_free_string_slots(pending_strings, in->n_cols);
+    free(pending_strings);
+    tf_batch_free(ob);
+    return TF_ERROR;
+}
+
+static int fill_down_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void fill_down_state_free(fill_down_state *st) {
+    if (st) {
+        if (st->cols) {
+            for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
+        }
+        free(st->cols);
+        if (st->last_vals) {
+            for (int i = 0; i < st->n_tracked; i++) free(st->last_vals[i]);
+            free(st->last_vals);
+        }
+        free(st->last_types); free(st->last_int);
+        free(st->last_float); free(st->last_bool);
+        free(st->last_date); free(st->last_timestamp);
+        free(st);
+    }
+}
+
+static void fill_down_destroy(tf_step *self) {
+    fill_down_state_free(self->state);
+    free(self);
+}
+
+tf_step *tf_fill_down_create(const cJSON *args) {
+    fill_down_state *st = calloc(1, sizeof(fill_down_state));
+    if (!st) return NULL;
+
+    if (args) {
+        cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
+        if (columns && cJSON_IsArray(columns)) {
+            int n = cJSON_GetArraySize(columns);
+            if (n > 0) {
+                st->cols = calloc(n, sizeof(char *));
+                if (!st->cols) { fill_down_state_free(st); return NULL; }
+                st->n_cols = (size_t)n;
+                for (int i = 0; i < n; i++) {
+                    cJSON *item = cJSON_GetArrayItem(columns, i);
+                    if (!cJSON_IsString(item) || !item->valuestring[0]) {
+                        fill_down_state_free(st);
+                        return NULL;
+                    }
+                    st->cols[i] = strdup(item->valuestring);
+                    if (!st->cols[i]) {
+                        fill_down_state_free(st);
+                        return NULL;
+                    }
+                }
+            }
+        }
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { fill_down_state_free(st); return NULL; }
+    step->process = fill_down_process;
+    step->flush = fill_down_flush;
+    step->destroy = fill_down_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_fill_null.c

@@ -0,0 +1,268 @@
+/*
+ * op_fill_null.c — Replace nulls with default values.
+ *
+ * Config: {"mapping": {"col1": "default1", "col2": "0"}}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include "date_utils.h"
+#include <stdlib.h>
+#include <string.h>
+
+/* Fill-null is row-local. Audit records are opt-in and capped so diagnostics
+ * cannot become an unbounded second output stream. */
+typedef struct {
+    char **col_names;
+    char **defaults;
+    size_t n;
+    size_t row_index;
+    size_t audit_limit;
+    size_t audit_emitted;
+    int audit;
+    tf_audit_options audit_opts;
+} fill_null_state;
+
+static int emit_fill_null_audit(fill_null_state *st, const tf_batch *b, size_t row,
+                                size_t col, size_t row_no, tf_side_channels *side) {
+    if (!st->audit || st->audit_emitted >= st->audit_limit || !side || !side->stats) return TF_OK;
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) return TF_ERROR;
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "audit") != TF_OK ||
+        tf_json_add_string(obj, "op", "fill-null") != TF_OK ||
+        tf_json_add_string(obj, "event", "null_filled") != TF_OK ||
+        tf_json_add_string(obj, "reason", "null_filled") != TF_OK ||
+        tf_json_add_string(obj, "channel", "audit") != TF_OK ||
+        tf_json_add_string(obj, "column", b->col_names[col] ? b->col_names[col] : "") != TF_OK ||
+        tf_json_add_number(obj, "row", (double)row_no) != TF_OK ||
+        tf_json_add_null(obj, "before") != TF_OK) {
+        goto done;
+    }
+    cJSON *after = tf_audit_cell_to_json(b, row, col, &st->audit_opts);
+    if (!after || tf_json_add_item(obj, "after", after) != TF_OK) goto done;
+    cJSON *row_obj = tf_audit_row_to_json(b, row, &st->audit_opts);
+    if (row_obj) {
+        if (tf_json_add_item(obj, "data", row_obj) != TF_OK) goto done;
+    } else if (st->audit_opts.include_row) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->stats, obj);
+done:
+    cJSON_Delete(obj);
+    if (rc == TF_OK) st->audit_emitted++;
+    return rc;
+}
+
+static int fill_null_process(tf_step *self, tf_batch *in, tf_batch **out,
+                             tf_side_channels *side) {
+    fill_null_state *st = self->state;
+    *out = NULL;
+    size_t row_base = st->row_index;
+
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    size_t *changed_cols = tf_callocarray_checked(st->n > 0 ? st->n : 1, sizeof(size_t));
+    if (!changed_cols) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            free(changed_cols);
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        size_t changed_count = 0;
+        for (size_t k = 0; k < st->n; k++) {
+            int ci = tf_batch_col_index(in, st->col_names[k]);
+            if (ci < 0 || !ob->nulls[ci] || !ob->nulls[ci][r]) continue;
+            const char *def = st->defaults[k];
+            int changed = 0;
+            switch (in->col_types[ci]) {
+                case TF_TYPE_STRING:
+                    if (tf_batch_set_string(ob, r, (size_t)ci, def) != TF_OK) {
+                        free(changed_cols);
+                        tf_batch_free(ob);
+                        return TF_ERROR;
+                    }
+                    changed = 1;
+                    break;
+                case TF_TYPE_INT64: {
+                    char *end;
+                    int64_t v = strtoll(def, &end, 10);
+                    if (*end == '\0') {
+                        if (tf_batch_set_int64(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
+                    break;
+                }
+                case TF_TYPE_FLOAT64: {
+                    char *end;
+                    double v = strtod(def, &end);
+                    if (*end == '\0') {
+                        if (tf_batch_set_float64(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
+                    break;
+                }
+                case TF_TYPE_BOOL:
+                    if (tf_batch_set_bool(ob, r, (size_t)ci, strcmp(def, "true") == 0) != TF_OK) {
+                        free(changed_cols);
+                        tf_batch_free(ob);
+                        return TF_ERROR;
+                    }
+                    changed = 1;
+                    break;
+                case TF_TYPE_DATE: {
+                    char *end;
+                    int32_t v = (int32_t)strtol(def, &end, 10);
+                    if (*end == '\0') {
+                        if (tf_batch_set_date(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
+                    break;
+                }
+                case TF_TYPE_TIMESTAMP: {
+                    char *end;
+                    int64_t v = strtoll(def, &end, 10);
+                    if (*end == '\0') {
+                        if (tf_batch_set_timestamp(ob, r, (size_t)ci, v) != TF_OK) {
+                            free(changed_cols);
+                            tf_batch_free(ob);
+                            return TF_ERROR;
+                        }
+                        changed = 1;
+                    }
+                    break;
+                }
+                default:
+                    break;
+            }
+            if (changed) changed_cols[changed_count++] = (size_t)ci;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            free(changed_cols);
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (changed_count > 0) {
+            size_t row_no = 0;
+            if (tf_size_add(row_base, r, &row_no) != TF_OK ||
+                tf_size_add(row_no, 1, &row_no) != TF_OK) {
+                free(changed_cols);
+                tf_batch_free(ob);
+                return TF_ERROR;
+            }
+            for (size_t i = 0; i < changed_count; i++) {
+                if (emit_fill_null_audit(st, ob, r, changed_cols[i], row_no, side) != TF_OK) {
+                    free(changed_cols);
+                    tf_batch_free(ob);
+                    return TF_ERROR;
+                }
+            }
+        }
+    }
+
+    if (tf_size_add(st->row_index, in->n_rows, &st->row_index) != TF_OK) {
+        free(changed_cols);
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    free(changed_cols);
+    *out = ob;
+    return TF_OK;
+}
+
+static int fill_null_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void fill_null_state_free(fill_null_state *st) {
+    if (!st) return;
+    tf_audit_options_free(&st->audit_opts);
+    if (st->col_names) {
+        for (size_t i = 0; i < st->n; i++) free(st->col_names[i]);
+    }
+    if (st->defaults) {
+        for (size_t i = 0; i < st->n; i++) free(st->defaults[i]);
+    }
+    free(st->col_names);
+    free(st->defaults);
+    free(st);
+}
+
+static void fill_null_destroy(tf_step *self) {
+    if (self) fill_null_state_free(self->state);
+    free(self);
+}
+
+tf_step *tf_fill_null_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *mapping = cJSON_GetObjectItemCaseSensitive(args, "mapping");
+    if (!mapping || !cJSON_IsObject(mapping)) return NULL;
+
+    int n = cJSON_GetArraySize(mapping);
+    fill_null_state *st = calloc(1, sizeof(fill_null_state));
+    if (!st) return NULL;
+    tf_audit_options_init(&st->audit_opts, 1);
+    st->col_names = tf_callocarray_checked(n > 0 ? (size_t)n : 1, sizeof(char *));
+    st->defaults = tf_callocarray_checked(n > 0 ? (size_t)n : 1, sizeof(char *));
+    st->n = (size_t)n;
+    st->audit_limit = 1000;
+    if (!st->col_names || !st->defaults) { fill_null_state_free(st); return NULL; }
+
+    int i = 0;
+    cJSON *entry = NULL;
+    cJSON_ArrayForEach(entry, mapping) {
+        if (!entry->string || !entry->string[0]) { fill_null_state_free(st); return NULL; }
+        st->col_names[i] = strdup(entry->string);
+        st->defaults[i] = strdup(cJSON_IsString(entry) ? entry->valuestring : "");
+        if (!st->col_names[i] || !st->defaults[i]) { fill_null_state_free(st); return NULL; }
+        i++;
+    }
+
+    cJSON *audit_j = cJSON_GetObjectItemCaseSensitive(args, "audit");
+    st->audit = cJSON_IsTrue(audit_j) ? 1 : 0;
+    cJSON *audit_limit_j = cJSON_GetObjectItemCaseSensitive(args, "audit_limit");
+    if (!audit_limit_j) audit_limit_j = cJSON_GetObjectItemCaseSensitive(args, "auditLimit");
+    if (audit_limit_j) {
+        size_t parsed_limit = 0;
+        if (tf_json_get_size_arg_any(args, "audit_limit", "auditLimit",
+                                     1, TF_MAX_AUDIT_RECORDS,
+                                     &parsed_limit, "fill-null") < 0) {
+            fill_null_state_free(st);
+            return NULL;
+        }
+        st->audit_limit = parsed_limit;
+    }
+
+    if (tf_audit_options_parse(&st->audit_opts, args, "fill-null") != TF_OK) {
+        fill_null_state_free(st);
+        return NULL;
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { fill_null_state_free(st); return NULL; }
+    step->process = fill_null_process;
+    step->flush = fill_null_flush;
+    step->destroy = fill_null_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_filter.c

@@ -0,0 +1,181 @@
+/*
+ * op_filter.c — Filter rows by expression.
+ *
+ * Config: {"expr": "col('age') > 25"}
+ * Evaluates expression for each row, keeps rows where result is true.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdbool.h>
+
+typedef struct {
+    tf_expr *expr;
+    char    *expr_text;
+    size_t   rows_in;
+    size_t   rows_out;
+    size_t   row_index;
+    size_t   audit_limit;
+    size_t   audit_emitted;
+    int      audit;
+    tf_audit_options audit_opts;
+} filter_state;
+
+static int filter_emit_drop_audit(filter_state *st, const tf_batch *b, size_t row,
+                                  tf_side_channels *side, const char *reason) {
+    if (!st->audit || st->audit_emitted >= st->audit_limit || !side || !side->stats) return TF_OK;
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) return TF_ERROR;
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "audit") != TF_OK ||
+        tf_json_add_string(obj, "op", "filter") != TF_OK ||
+        tf_json_add_string(obj, "event", "row_dropped") != TF_OK ||
+        tf_json_add_string(obj, "reason", reason ? reason : "predicate_false") != TF_OK ||
+        tf_json_add_string(obj, "channel", "audit") != TF_OK ||
+        tf_json_add_string(obj, "expr", st->expr_text ? st->expr_text : "") != TF_OK ||
+        tf_json_add_number(obj, "row", (double)st->row_index) != TF_OK) {
+        goto done;
+    }
+    cJSON *data = tf_audit_row_to_json(b, row, &st->audit_opts);
+    if (data) {
+        if (tf_json_add_item(obj, "data", data) != TF_OK) goto done;
+    } else if (st->audit_opts.include_row) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->stats, obj);
+done:
+    cJSON_Delete(obj);
+    if (rc == TF_OK) st->audit_emitted++;
+    return rc;
+}
+
+static int filter_process(tf_step *self, tf_batch *in, tf_batch **out,
+                          tf_side_channels *side) {
+    filter_state *st = self->state;
+    *out = NULL;
+
+    /* Create output batch with same schema */
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    /* Filter rows */
+    size_t out_row = 0;
+    for (size_t r = 0; r < in->n_rows; r++) {
+        st->row_index++;
+        bool keep = false;
+        int eval_ok = tf_expr_eval(st->expr, in, r, &keep) == TF_OK;
+        if (!eval_ok || !keep) {
+            if (filter_emit_drop_audit(st, in, r, side,
+                                       eval_ok ? "predicate_false" : "predicate_error") != TF_OK) {
+                tf_batch_free(ob);
+                return TF_ERROR;
+            }
+            continue;
+        }
+
+        if (tf_batch_copy_row(ob, out_row, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, out_row) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        out_row++;
+    }
+
+    st->rows_in += in->n_rows;
+    st->rows_out += out_row;
+
+    /* Emit stats to side channel */
+    if (side && side->stats) {
+        char stats_buf[128];
+        snprintf(stats_buf, sizeof(stats_buf),
+                 "{\"op\":\"filter\",\"rows_in\":%zu,\"rows_out\":%zu}",
+                 in->n_rows, out_row);
+        if (tf_buffer_write_line(side->stats, stats_buf) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+
+    if (out_row > 0) {
+        *out = ob;
+    } else {
+        tf_batch_free(ob);
+    }
+    return TF_OK;
+}
+
+static int filter_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side;
+    *out = NULL;
+    return TF_OK;
+}
+
+static void filter_destroy(tf_step *self) {
+    filter_state *st = self->state;
+    if (st) {
+        tf_expr_free(st->expr);
+        free(st->expr_text);
+        tf_audit_options_free(&st->audit_opts);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_filter_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *expr_json = cJSON_GetObjectItemCaseSensitive(args, "expr");
+    if (!cJSON_IsString(expr_json)) return NULL;
+
+    tf_expr *expr = tf_expr_parse(expr_json->valuestring);
+    if (!expr) return NULL;
+
+    filter_state *st = calloc(1, sizeof(filter_state));
+    if (!st) { tf_expr_free(expr); return NULL; }
+    st->expr = expr;
+    st->expr_text = strdup(expr_json->valuestring);
+    st->audit_limit = 1000;
+    tf_audit_options_init(&st->audit_opts, 1);
+    cJSON *audit_json = cJSON_GetObjectItemCaseSensitive(args, "audit");
+    st->audit = cJSON_IsTrue(audit_json) ? 1 : 0;
+    cJSON *audit_limit_json = cJSON_GetObjectItemCaseSensitive(args, "audit_limit");
+    if (!audit_limit_json) audit_limit_json = cJSON_GetObjectItemCaseSensitive(args, "auditLimit");
+    if (audit_limit_json) {
+        size_t parsed_limit = 0;
+        if (tf_json_get_size_arg_any(args, "audit_limit", "auditLimit",
+                                     1, TF_MAX_AUDIT_RECORDS,
+                                     &parsed_limit, "filter") < 0) {
+            tf_expr_free(expr);
+            free(st->expr_text);
+            tf_audit_options_free(&st->audit_opts);
+            free(st);
+            return NULL;
+        }
+        st->audit_limit = parsed_limit;
+    }
+    if (tf_audit_options_parse(&st->audit_opts, args, "filter") != TF_OK) {
+        tf_expr_free(expr);
+        free(st->expr_text);
+        tf_audit_options_free(&st->audit_opts);
+        free(st);
+        return NULL;
+    }
+    if (!st->expr_text) { tf_expr_free(expr); tf_audit_options_free(&st->audit_opts); free(st); return NULL; }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { tf_expr_free(expr); free(st->expr_text); tf_audit_options_free(&st->audit_opts); free(st); return NULL; }
+    step->process = filter_process;
+    step->flush = filter_flush;
+    step->destroy = filter_destroy;
+    step->state = st;
+    return step;
+}