tranfi 0.1.2 → 0.2.1

package/csrc/op_filter.c

@@ -10,13 +10,48 @@
 #include <stdlib.h>
 #include <string.h>
 #include <stdio.h>
+#include <stdbool.h>
 
 typedef struct {
     tf_expr *expr;
+    char    *expr_text;
     size_t   rows_in;
     size_t   rows_out;
+    size_t   row_index;
+    size_t   audit_limit;
+    size_t   audit_emitted;
+    int      audit;
+    tf_audit_options audit_opts;
 } filter_state;
 
+static int filter_emit_drop_audit(filter_state *st, const tf_batch *b, size_t row,
+                                  tf_side_channels *side, const char *reason) {
+    if (!st->audit || st->audit_emitted >= st->audit_limit || !side || !side->stats) return TF_OK;
+    cJSON *obj = cJSON_CreateObject();
+    if (!obj) return TF_ERROR;
+    int rc = TF_ERROR;
+    if (tf_json_add_string(obj, "type", "audit") != TF_OK ||
+        tf_json_add_string(obj, "op", "filter") != TF_OK ||
+        tf_json_add_string(obj, "event", "row_dropped") != TF_OK ||
+        tf_json_add_string(obj, "reason", reason ? reason : "predicate_false") != TF_OK ||
+        tf_json_add_string(obj, "channel", "audit") != TF_OK ||
+        tf_json_add_string(obj, "expr", st->expr_text ? st->expr_text : "") != TF_OK ||
+        tf_json_add_number(obj, "row", (double)st->row_index) != TF_OK) {
+        goto done;
+    }
+    cJSON *data = tf_audit_row_to_json(b, row, &st->audit_opts);
+    if (data) {
+        if (tf_json_add_item(obj, "data", data) != TF_OK) goto done;
+    } else if (st->audit_opts.include_row) {
+        goto done;
+    }
+    rc = tf_buffer_write_json_line(side->stats, obj);
+done:
+    cJSON_Delete(obj);
+    if (rc == TF_OK) st->audit_emitted++;
+    return rc;
+}
+
 static int filter_process(tf_step *self, tf_batch *in, tf_batch **out,
                           tf_side_channels *side) {
     filter_state *st = self->state;
@@ -25,55 +60,36 @@ static int filter_process(tf_step *self, tf_batch *in, tf_batch **out,
     /* Create output batch with same schema */
     tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
     if (!ob) return TF_ERROR;
-    for (size_t i = 0; i < in->n_cols; i++) {
-        tf_batch_set_schema(ob, i, in->col_names[i], in->col_types[i]);
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
     }
 
     /* Filter rows */
     size_t out_row = 0;
     for (size_t r = 0; r < in->n_rows; r++) {
+        st->row_index++;
         bool keep = false;
-        if (tf_expr_eval(st->expr, in, r, &keep) != TF_OK) continue;
-        if (!keep) continue;
+        int eval_ok = tf_expr_eval(st->expr, in, r, &keep) == TF_OK;
+        if (!eval_ok || !keep) {
+            if (filter_emit_drop_audit(st, in, r, side,
+                                       eval_ok ? "predicate_false" : "predicate_error") != TF_OK) {
+                tf_batch_free(ob);
+                return TF_ERROR;
+            }
+            continue;
+        }
 
-        if (tf_batch_ensure_capacity(ob, out_row + 1) != TF_OK) {
+        if (tf_batch_copy_row(ob, out_row, in, r) != TF_OK) {
             tf_batch_free(ob);
             return TF_ERROR;
         }
-
-        /* Copy row */
-        for (size_t c = 0; c < in->n_cols; c++) {
-            if (tf_batch_is_null(in, r, c)) {
-                tf_batch_set_null(ob, out_row, c);
-                continue;
-            }
-            switch (in->col_types[c]) {
-                case TF_TYPE_BOOL:
-                    tf_batch_set_bool(ob, out_row, c, tf_batch_get_bool(in, r, c));
-                    break;
-                case TF_TYPE_INT64:
-                    tf_batch_set_int64(ob, out_row, c, tf_batch_get_int64(in, r, c));
-                    break;
-                case TF_TYPE_FLOAT64:
-                    tf_batch_set_float64(ob, out_row, c, tf_batch_get_float64(in, r, c));
-                    break;
-                case TF_TYPE_STRING:
-                    tf_batch_set_string(ob, out_row, c, tf_batch_get_string(in, r, c));
-                    break;
-                case TF_TYPE_DATE:
-                    tf_batch_set_date(ob, out_row, c, tf_batch_get_date(in, r, c));
-                    break;
-                case TF_TYPE_TIMESTAMP:
-                    tf_batch_set_timestamp(ob, out_row, c, tf_batch_get_timestamp(in, r, c));
-                    break;
-                default:
-                    tf_batch_set_null(ob, out_row, c);
-                    break;
-            }
+        if (tf_batch_expose_row(ob, out_row) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
         }
         out_row++;
     }
-    ob->n_rows = out_row;
 
     st->rows_in += in->n_rows;
     st->rows_out += out_row;
@@ -82,9 +98,12 @@ static int filter_process(tf_step *self, tf_batch *in, tf_batch **out,
     if (side && side->stats) {
         char stats_buf[128];
         snprintf(stats_buf, sizeof(stats_buf),
-                 "{\"op\":\"filter\",\"rows_in\":%zu,\"rows_out\":%zu}\n",
+                 "{\"op\":\"filter\",\"rows_in\":%zu,\"rows_out\":%zu}",
                  in->n_rows, out_row);
-        tf_buffer_write_str(side->stats, stats_buf);
+        if (tf_buffer_write_line(side->stats, stats_buf) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
     }
 
     if (out_row > 0) {
@@ -105,6 +124,8 @@ static void filter_destroy(tf_step *self) {
     filter_state *st = self->state;
     if (st) {
         tf_expr_free(st->expr);
+        free(st->expr_text);
+        tf_audit_options_free(&st->audit_opts);
         free(st);
     }
     free(self);
@@ -121,9 +142,37 @@ tf_step *tf_filter_create(const cJSON *args) {
     filter_state *st = calloc(1, sizeof(filter_state));
     if (!st) { tf_expr_free(expr); return NULL; }
     st->expr = expr;
+    st->expr_text = strdup(expr_json->valuestring);
+    st->audit_limit = 1000;
+    tf_audit_options_init(&st->audit_opts, 1);
+    cJSON *audit_json = cJSON_GetObjectItemCaseSensitive(args, "audit");
+    st->audit = cJSON_IsTrue(audit_json) ? 1 : 0;
+    cJSON *audit_limit_json = cJSON_GetObjectItemCaseSensitive(args, "audit_limit");
+    if (!audit_limit_json) audit_limit_json = cJSON_GetObjectItemCaseSensitive(args, "auditLimit");
+    if (audit_limit_json) {
+        size_t parsed_limit = 0;
+        if (tf_json_get_size_arg_any(args, "audit_limit", "auditLimit",
+                                     1, TF_MAX_AUDIT_RECORDS,
+                                     &parsed_limit, "filter") < 0) {
+            tf_expr_free(expr);
+            free(st->expr_text);
+            tf_audit_options_free(&st->audit_opts);
+            free(st);
+            return NULL;
+        }
+        st->audit_limit = parsed_limit;
+    }
+    if (tf_audit_options_parse(&st->audit_opts, args, "filter") != TF_OK) {
+        tf_expr_free(expr);
+        free(st->expr_text);
+        tf_audit_options_free(&st->audit_opts);
+        free(st);
+        return NULL;
+    }
+    if (!st->expr_text) { tf_expr_free(expr); tf_audit_options_free(&st->audit_opts); free(st); return NULL; }
 
-    tf_step *step = malloc(sizeof(tf_step));
-    if (!step) { tf_expr_free(expr); free(st); return NULL; }
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { tf_expr_free(expr); free(st->expr_text); tf_audit_options_free(&st->audit_opts); free(st); return NULL; }
     step->process = filter_process;
     step->flush = filter_flush;
     step->destroy = filter_destroy;