tranfi 0.0.1 → 0.1.2

package/csrc/op_explode.c

@@ -0,0 +1,108 @@
+/*
+ * op_explode.c — Split delimited string into multiple rows.
+ *
+ * Config: {"column": "tags", "delimiter": ","}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    char *column;
+    char *delimiter;
+} explode_state;
+
+static int explode_process(tf_step *self, tf_batch *in, tf_batch **out,
+                           tf_side_channels *side) {
+    (void)side;
+    explode_state *st = self->state;
+    *out = NULL;
+
+    int ci = tf_batch_col_index(in, st->column);
+
+    /* Estimate max output rows */
+    size_t max_out = in->n_rows * 4;
+    tf_batch *ob = tf_batch_create(in->n_cols, max_out > 0 ? max_out : 16);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+
+    size_t out_row = 0;
+    size_t delim_len = strlen(st->delimiter);
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (ci < 0 || tf_batch_is_null(in, r, ci) || in->col_types[ci] != TF_TYPE_STRING) {
+            tf_batch_copy_row(ob, out_row, in, r);
+            ob->n_rows = ++out_row;
+            continue;
+        }
+
+        const char *val = tf_batch_get_string(in, r, ci);
+        const char *p = val;
+
+        while (*p) {
+            const char *found = strstr(p, st->delimiter);
+            size_t tok_len = found ? (size_t)(found - p) : strlen(p);
+
+            tf_batch_copy_row(ob, out_row, in, r);
+            /* Override the exploded column */
+            char *tok = malloc(tok_len + 1);
+            if (tok) {
+                memcpy(tok, p, tok_len);
+                tok[tok_len] = '\0';
+                /* Trim leading/trailing whitespace */
+                char *s = tok;
+                while (*s == ' ') s++;
+                char *e = s + strlen(s);
+                while (e > s && *(e - 1) == ' ') e--;
+                *e = '\0';
+                tf_batch_set_string(ob, out_row, ci, s);
+                free(tok);
+            }
+            ob->n_rows = ++out_row;
+
+            if (found) p = found + delim_len;
+            else break;
+        }
+    }
+
+    if (out_row > 0) {
+        *out = ob;
+    } else {
+        tf_batch_free(ob);
+    }
+    return TF_OK;
+}
+
+static int explode_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void explode_destroy(tf_step *self) {
+    explode_state *st = self->state;
+    if (st) { free(st->column); free(st->delimiter); free(st); }
+    free(self);
+}
+
+tf_step *tf_explode_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    if (!cJSON_IsString(col_j)) return NULL;
+
+    explode_state *st = calloc(1, sizeof(explode_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+
+    cJSON *delim_j = cJSON_GetObjectItemCaseSensitive(args, "delimiter");
+    st->delimiter = strdup(cJSON_IsString(delim_j) ? delim_j->valuestring : ",");
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st->column); free(st->delimiter); free(st); return NULL; }
+    step->process = explode_process;
+    step->flush = explode_flush;
+    step->destroy = explode_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_fill_down.c

@@ -0,0 +1,163 @@
+/*
+ * op_fill_down.c — Forward-fill nulls with last non-null value.
+ *
+ * Config: {"columns": ["city", "state"]}
+ *   or {} for all columns.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include "date_utils.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char  **cols;
+    size_t  n_cols;
+    /* Last non-null value per column (stored as strings for simplicity) */
+    char  **last_vals;
+    int    *last_types; /* tf_type */
+    int64_t *last_int;
+    double  *last_float;
+    int     *last_bool;
+    int32_t *last_date;
+    int64_t *last_timestamp;
+    int      n_tracked;
+    int      initialized;
+} fill_down_state;
+
+static int fill_down_process(tf_step *self, tf_batch *in, tf_batch **out,
+                             tf_side_channels *side) {
+    (void)side;
+    fill_down_state *st = self->state;
+    *out = NULL;
+
+    if (!st->initialized) {
+        st->n_tracked = (int)in->n_cols;
+        st->last_vals = calloc(in->n_cols, sizeof(char *));
+        st->last_types = calloc(in->n_cols, sizeof(int));
+        st->last_int = calloc(in->n_cols, sizeof(int64_t));
+        st->last_float = calloc(in->n_cols, sizeof(double));
+        st->last_bool = calloc(in->n_cols, sizeof(int));
+        st->last_date = calloc(in->n_cols, sizeof(int32_t));
+        st->last_timestamp = calloc(in->n_cols, sizeof(int64_t));
+        for (size_t c = 0; c < in->n_cols; c++) st->last_types[c] = -1; /* no value yet */
+        st->initialized = 1;
+    }
+
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        tf_batch_copy_row(ob, r, in, r);
+
+        for (size_t c = 0; c < in->n_cols; c++) {
+            /* Check if this column is targeted */
+            int target = 0;
+            if (st->n_cols == 0) {
+                target = 1;
+            } else {
+                for (size_t k = 0; k < st->n_cols; k++) {
+                    if (strcmp(in->col_names[c], st->cols[k]) == 0) { target = 1; break; }
+                }
+            }
+            if (!target) continue;
+
+            if (!tf_batch_is_null(in, r, c)) {
+                /* Update last value */
+                st->last_types[c] = (int)in->col_types[c];
+                switch (in->col_types[c]) {
+                    case TF_TYPE_STRING:
+                        free(st->last_vals[c]);
+                        st->last_vals[c] = strdup(tf_batch_get_string(in, r, c));
+                        break;
+                    case TF_TYPE_INT64:
+                        st->last_int[c] = tf_batch_get_int64(in, r, c); break;
+                    case TF_TYPE_FLOAT64:
+                        st->last_float[c] = tf_batch_get_float64(in, r, c); break;
+                    case TF_TYPE_BOOL:
+                        st->last_bool[c] = tf_batch_get_bool(in, r, c); break;
+                    case TF_TYPE_DATE:
+                        st->last_date[c] = tf_batch_get_date(in, r, c); break;
+                    case TF_TYPE_TIMESTAMP:
+                        st->last_timestamp[c] = tf_batch_get_timestamp(in, r, c); break;
+                    default: break;
+                }
+            } else if (st->last_types[c] >= 0) {
+                /* Fill with last value */
+                switch (in->col_types[c]) {
+                    case TF_TYPE_STRING:
+                        if (st->last_vals[c]) tf_batch_set_string(ob, r, c, st->last_vals[c]);
+                        break;
+                    case TF_TYPE_INT64:
+                        tf_batch_set_int64(ob, r, c, st->last_int[c]); break;
+                    case TF_TYPE_FLOAT64:
+                        tf_batch_set_float64(ob, r, c, st->last_float[c]); break;
+                    case TF_TYPE_BOOL:
+                        tf_batch_set_bool(ob, r, c, st->last_bool[c]); break;
+                    case TF_TYPE_DATE:
+                        tf_batch_set_date(ob, r, c, st->last_date[c]); break;
+                    case TF_TYPE_TIMESTAMP:
+                        tf_batch_set_timestamp(ob, r, c, st->last_timestamp[c]); break;
+                    default: break;
+                }
+            }
+        }
+        ob->n_rows = r + 1;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int fill_down_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void fill_down_destroy(tf_step *self) {
+    fill_down_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
+        free(st->cols);
+        if (st->last_vals) {
+            for (int i = 0; i < st->n_tracked; i++) free(st->last_vals[i]);
+            free(st->last_vals);
+        }
+        free(st->last_types); free(st->last_int);
+        free(st->last_float); free(st->last_bool);
+        free(st->last_date); free(st->last_timestamp);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_fill_down_create(const cJSON *args) {
+    fill_down_state *st = calloc(1, sizeof(fill_down_state));
+    if (!st) return NULL;
+
+    if (args) {
+        cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
+        if (columns && cJSON_IsArray(columns)) {
+            int n = cJSON_GetArraySize(columns);
+            if (n > 0) {
+                st->cols = calloc(n, sizeof(char *));
+                st->n_cols = n;
+                for (int i = 0; i < n; i++) {
+                    cJSON *item = cJSON_GetArrayItem(columns, i);
+                    if (cJSON_IsString(item)) st->cols[i] = strdup(item->valuestring);
+                }
+            }
+        }
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st); return NULL; }
+    step->process = fill_down_process;
+    step->flush = fill_down_flush;
+    step->destroy = fill_down_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_fill_null.c

@@ -0,0 +1,123 @@
+/*
+ * op_fill_null.c — Replace nulls with default values.
+ *
+ * Config: {"mapping": {"col1": "default1", "col2": "0"}}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include "date_utils.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    char **col_names;
+    char **defaults;
+    size_t n;
+} fill_null_state;
+
+static int fill_null_process(tf_step *self, tf_batch *in, tf_batch **out,
+                             tf_side_channels *side) {
+    (void)side;
+    fill_null_state *st = self->state;
+    *out = NULL;
+
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    for (size_t r = 0; r < in->n_rows; r++) {
+        tf_batch_copy_row(ob, r, in, r);
+        ob->n_rows = r + 1;
+    }
+
+    /* Fill nulls */
+    for (size_t k = 0; k < st->n; k++) {
+        int ci = tf_batch_col_index(ob, st->col_names[k]);
+        if (ci < 0) continue;
+        for (size_t r = 0; r < ob->n_rows; r++) {
+            if (!tf_batch_is_null(ob, r, ci)) continue;
+            const char *def = st->defaults[k];
+            switch (ob->col_types[ci]) {
+                case TF_TYPE_STRING:
+                    tf_batch_set_string(ob, r, ci, def);
+                    break;
+                case TF_TYPE_INT64: {
+                    char *end;
+                    int64_t v = strtoll(def, &end, 10);
+                    if (*end == '\0') tf_batch_set_int64(ob, r, ci, v);
+                    break;
+                }
+                case TF_TYPE_FLOAT64: {
+                    char *end;
+                    double v = strtod(def, &end);
+                    if (*end == '\0') tf_batch_set_float64(ob, r, ci, v);
+                    break;
+                }
+                case TF_TYPE_BOOL:
+                    tf_batch_set_bool(ob, r, ci, strcmp(def, "true") == 0);
+                    break;
+                case TF_TYPE_DATE: {
+                    char *end;
+                    int32_t v = (int32_t)strtol(def, &end, 10);
+                    if (*end == '\0') tf_batch_set_date(ob, r, ci, v);
+                    break;
+                }
+                case TF_TYPE_TIMESTAMP: {
+                    char *end;
+                    int64_t v = strtoll(def, &end, 10);
+                    if (*end == '\0') tf_batch_set_timestamp(ob, r, ci, v);
+                    break;
+                }
+                default:
+                    break;
+            }
+        }
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int fill_null_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void fill_null_destroy(tf_step *self) {
+    fill_null_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n; i++) { free(st->col_names[i]); free(st->defaults[i]); }
+        free(st->col_names); free(st->defaults);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_fill_null_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *mapping = cJSON_GetObjectItemCaseSensitive(args, "mapping");
+    if (!mapping || !cJSON_IsObject(mapping)) return NULL;
+
+    int n = cJSON_GetArraySize(mapping);
+    fill_null_state *st = calloc(1, sizeof(fill_null_state));
+    if (!st) return NULL;
+    st->col_names = calloc(n, sizeof(char *));
+    st->defaults = calloc(n, sizeof(char *));
+    st->n = n;
+
+    int i = 0;
+    cJSON *entry = NULL;
+    cJSON_ArrayForEach(entry, mapping) {
+        st->col_names[i] = strdup(entry->string);
+        st->defaults[i] = strdup(cJSON_IsString(entry) ? entry->valuestring : "");
+        i++;
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { fill_null_destroy(&(tf_step){.state = st}); return NULL; }
+    step->process = fill_null_process;
+    step->flush = fill_null_flush;
+    step->destroy = fill_null_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_filter.c

@@ -0,0 +1,132 @@
+/*
+ * op_filter.c — Filter rows by expression.
+ *
+ * Config: {"expr": "col('age') > 25"}
+ * Evaluates expression for each row, keeps rows where result is true.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    tf_expr *expr;
+    size_t   rows_in;
+    size_t   rows_out;
+} filter_state;
+
+static int filter_process(tf_step *self, tf_batch *in, tf_batch **out,
+                          tf_side_channels *side) {
+    filter_state *st = self->state;
+    *out = NULL;
+
+    /* Create output batch with same schema */
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t i = 0; i < in->n_cols; i++) {
+        tf_batch_set_schema(ob, i, in->col_names[i], in->col_types[i]);
+    }
+
+    /* Filter rows */
+    size_t out_row = 0;
+    for (size_t r = 0; r < in->n_rows; r++) {
+        bool keep = false;
+        if (tf_expr_eval(st->expr, in, r, &keep) != TF_OK) continue;
+        if (!keep) continue;
+
+        if (tf_batch_ensure_capacity(ob, out_row + 1) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+
+        /* Copy row */
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (tf_batch_is_null(in, r, c)) {
+                tf_batch_set_null(ob, out_row, c);
+                continue;
+            }
+            switch (in->col_types[c]) {
+                case TF_TYPE_BOOL:
+                    tf_batch_set_bool(ob, out_row, c, tf_batch_get_bool(in, r, c));
+                    break;
+                case TF_TYPE_INT64:
+                    tf_batch_set_int64(ob, out_row, c, tf_batch_get_int64(in, r, c));
+                    break;
+                case TF_TYPE_FLOAT64:
+                    tf_batch_set_float64(ob, out_row, c, tf_batch_get_float64(in, r, c));
+                    break;
+                case TF_TYPE_STRING:
+                    tf_batch_set_string(ob, out_row, c, tf_batch_get_string(in, r, c));
+                    break;
+                case TF_TYPE_DATE:
+                    tf_batch_set_date(ob, out_row, c, tf_batch_get_date(in, r, c));
+                    break;
+                case TF_TYPE_TIMESTAMP:
+                    tf_batch_set_timestamp(ob, out_row, c, tf_batch_get_timestamp(in, r, c));
+                    break;
+                default:
+                    tf_batch_set_null(ob, out_row, c);
+                    break;
+            }
+        }
+        out_row++;
+    }
+    ob->n_rows = out_row;
+
+    st->rows_in += in->n_rows;
+    st->rows_out += out_row;
+
+    /* Emit stats to side channel */
+    if (side && side->stats) {
+        char stats_buf[128];
+        snprintf(stats_buf, sizeof(stats_buf),
+                 "{\"op\":\"filter\",\"rows_in\":%zu,\"rows_out\":%zu}\n",
+                 in->n_rows, out_row);
+        tf_buffer_write_str(side->stats, stats_buf);
+    }
+
+    if (out_row > 0) {
+        *out = ob;
+    } else {
+        tf_batch_free(ob);
+    }
+    return TF_OK;
+}
+
+static int filter_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side;
+    *out = NULL;
+    return TF_OK;
+}
+
+static void filter_destroy(tf_step *self) {
+    filter_state *st = self->state;
+    if (st) {
+        tf_expr_free(st->expr);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_filter_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *expr_json = cJSON_GetObjectItemCaseSensitive(args, "expr");
+    if (!cJSON_IsString(expr_json)) return NULL;
+
+    tf_expr *expr = tf_expr_parse(expr_json->valuestring);
+    if (!expr) return NULL;
+
+    filter_state *st = calloc(1, sizeof(filter_state));
+    if (!st) { tf_expr_free(expr); return NULL; }
+    st->expr = expr;
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { tf_expr_free(expr); free(st); return NULL; }
+    step->process = filter_process;
+    step->flush = filter_flush;
+    step->destroy = filter_destroy;
+    step->state = st;
+    return step;
+}