tranfi 0.0.2 → 0.1.2

package/csrc/op_lead.c

@@ -0,0 +1,190 @@
+/*
+ * op_lead.c — Lookahead: access value N rows ahead.
+ *
+ * Config: {"column": "price", "offset": 1, "result": "next_price"}
+ *
+ * Buffers the last `offset` rows across batch boundaries.
+ * On flush, emits remaining rows with NULL lead values.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char   *column;
+    char   *result;
+    size_t  offset;
+    /* Pending rows: stored as a batch that hasn't been emitted yet */
+    tf_batch *pending;
+} lead_state;
+
+static double get_numeric(const tf_batch *b, size_t r, int ci) {
+    if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
+    if (b->col_types[ci] == TF_TYPE_FLOAT64) return tf_batch_get_float64(b, r, ci);
+    return 0;
+}
+
+static int lead_process(tf_step *self, tf_batch *in, tf_batch **out,
+                        tf_side_channels *side) {
+    (void)side;
+    lead_state *st = self->state;
+    *out = NULL;
+
+    size_t pend_count = st->pending ? st->pending->n_rows : 0;
+    size_t total = pend_count + in->n_rows;
+
+    if (total <= st->offset) {
+        /* Not enough rows yet — append all to pending */
+        tf_batch *new_pend = tf_batch_create(in->n_cols, total);
+        if (!new_pend) return TF_ERROR;
+        for (size_t c = 0; c < in->n_cols; c++)
+            tf_batch_set_schema(new_pend, c, in->col_names[c], in->col_types[c]);
+        size_t row = 0;
+        if (st->pending) {
+            for (size_t r = 0; r < pend_count; r++)
+                tf_batch_copy_row(new_pend, row++, st->pending, r);
+            tf_batch_free(st->pending);
+        }
+        for (size_t r = 0; r < in->n_rows; r++)
+            tf_batch_copy_row(new_pend, row++, in, r);
+        new_pend->n_rows = total;
+        st->pending = new_pend;
+        return TF_OK;
+    }
+
+    size_t emit_count = total - st->offset;
+    tf_batch *ob = tf_batch_create(in->n_cols + 1, emit_count);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    tf_batch_set_schema(ob, in->n_cols, st->result, TF_TYPE_FLOAT64);
+
+    /* Emit rows: for each emitted row i, lead value comes from row i+offset */
+    for (size_t i = 0; i < emit_count; i++) {
+        /* Source row for base data */
+        if (i < pend_count) {
+            tf_batch_copy_row(ob, i, st->pending, i);
+        } else {
+            tf_batch_copy_row(ob, i, in, i - pend_count);
+        }
+
+        /* Lead value from row i + offset */
+        size_t lead_idx = i + st->offset;
+        tf_batch *lead_src;
+        size_t lead_row;
+        if (lead_idx < pend_count) {
+            lead_src = st->pending;
+            lead_row = lead_idx;
+        } else {
+            lead_src = in;
+            lead_row = lead_idx - pend_count;
+        }
+
+        int lead_ci = tf_batch_col_index(lead_src, st->column);
+        if (lead_ci >= 0 && !tf_batch_is_null(lead_src, lead_row, lead_ci)) {
+            tf_batch_set_float64(ob, i, in->n_cols, get_numeric(lead_src, lead_row, lead_ci));
+        } else {
+            tf_batch_set_null(ob, i, in->n_cols);
+        }
+    }
+    ob->n_rows = emit_count;
+
+    /* Store remaining rows as new pending */
+    if (st->pending) {
+        tf_batch_free(st->pending);
+        st->pending = NULL;
+    }
+    size_t new_pend_count = st->offset;
+    if (new_pend_count > 0) {
+        tf_batch *new_pend = tf_batch_create(in->n_cols, new_pend_count);
+        if (!new_pend) { tf_batch_free(ob); return TF_ERROR; }
+        for (size_t c = 0; c < in->n_cols; c++)
+            tf_batch_set_schema(new_pend, c, in->col_names[c], in->col_types[c]);
+        for (size_t i = 0; i < new_pend_count; i++) {
+            size_t src_idx = total - st->offset + i;
+            if (src_idx < pend_count) {
+                /* This shouldn't happen since we emit at least pend_count rows when total > offset */
+                tf_batch_copy_row(new_pend, i, st->pending, src_idx);
+            } else {
+                tf_batch_copy_row(new_pend, i, in, src_idx - pend_count);
+            }
+        }
+        new_pend->n_rows = new_pend_count;
+        st->pending = new_pend;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int lead_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)side;
+    lead_state *st = self->state;
+    *out = NULL;
+
+    if (!st->pending || st->pending->n_rows == 0) return TF_OK;
+
+    /* Emit remaining pending rows with NULL lead values */
+    tf_batch *pend = st->pending;
+    tf_batch *ob = tf_batch_create(pend->n_cols + 1, pend->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < pend->n_cols; c++)
+        tf_batch_set_schema(ob, c, pend->col_names[c], pend->col_types[c]);
+    tf_batch_set_schema(ob, pend->n_cols, st->result, TF_TYPE_FLOAT64);
+
+    for (size_t r = 0; r < pend->n_rows; r++) {
+        tf_batch_copy_row(ob, r, pend, r);
+        tf_batch_set_null(ob, r, pend->n_cols);
+    }
+    ob->n_rows = pend->n_rows;
+
+    tf_batch_free(st->pending);
+    st->pending = NULL;
+
+    *out = ob;
+    return TF_OK;
+}
+
+static void lead_destroy(tf_step *self) {
+    lead_state *st = self->state;
+    if (st) {
+        free(st->column);
+        free(st->result);
+        if (st->pending) tf_batch_free(st->pending);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_lead_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    if (!cJSON_IsString(col_j)) return NULL;
+
+    lead_state *st = calloc(1, sizeof(lead_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+
+    cJSON *off_j = cJSON_GetObjectItemCaseSensitive(args, "offset");
+    st->offset = (cJSON_IsNumber(off_j) && off_j->valueint > 0) ? (size_t)off_j->valueint : 1;
+
+    cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
+    if (cJSON_IsString(res_j)) {
+        st->result = strdup(res_j->valuestring);
+    } else {
+        char buf[256];
+        snprintf(buf, sizeof(buf), "%s_lead", st->column);
+        st->result = strdup(buf);
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st->column); free(st->result); free(st); return NULL; }
+    step->process = lead_process;
+    step->flush = lead_flush;
+    step->destroy = lead_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_normalize.c

@@ -0,0 +1,226 @@
+/*
+ * op_normalize.c — Min-max or z-score normalization.
+ * Aggregate op: buffers all rows, computes stats, then emits normalized.
+ *
+ * Config: {"columns": ["price", "score"], "method": "minmax"}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <math.h>
+
+typedef enum {
+    NORM_MINMAX,
+    NORM_ZSCORE
+} norm_method;
+
+typedef struct {
+    int     col_idx;
+    /* Welford's online stats */
+    size_t  count;
+    double  mean;
+    double  m2;
+    double  min_val;
+    double  max_val;
+} col_stats;
+
+/* Row buffer entry */
+typedef struct {
+    tf_batch *batch; /* single-row batch */
+} buf_row;
+
+typedef struct {
+    char       **columns;
+    size_t       n_columns;
+    norm_method  method;
+    col_stats   *stats;
+    buf_row     *rows;
+    size_t       n_rows;
+    size_t       cap_rows;
+    int          has_schema;
+    size_t       schema_n_cols;
+    char       **schema_names;
+    tf_type     *schema_types;
+} normalize_state;
+
+static norm_method parse_method(const char *s) {
+    if (s && strcmp(s, "zscore") == 0) return NORM_ZSCORE;
+    return NORM_MINMAX;
+}
+
+static double get_numeric(const tf_batch *b, size_t r, int ci) {
+    if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
+    if (b->col_types[ci] == TF_TYPE_FLOAT64) return tf_batch_get_float64(b, r, ci);
+    return 0;
+}
+
+static void add_row(normalize_state *st, tf_batch *b, size_t r) {
+    if (st->n_rows >= st->cap_rows) {
+        size_t newcap = st->cap_rows ? st->cap_rows * 2 : 256;
+        buf_row *tmp = realloc(st->rows, newcap * sizeof(buf_row));
+        if (!tmp) return;
+        st->rows = tmp;
+        st->cap_rows = newcap;
+    }
+    /* Copy single row into its own batch */
+    tf_batch *rb = tf_batch_create(b->n_cols, 1);
+    if (!rb) return;
+    for (size_t c = 0; c < b->n_cols; c++)
+        tf_batch_set_schema(rb, c, b->col_names[c], b->col_types[c]);
+    tf_batch_copy_row(rb, 0, b, r);
+    rb->n_rows = 1;
+    st->rows[st->n_rows].batch = rb;
+    st->n_rows++;
+}
+
+static int normalize_process(tf_step *self, tf_batch *in, tf_batch **out,
+                             tf_side_channels *side) {
+    (void)side;
+    normalize_state *st = self->state;
+    *out = NULL;
+
+    /* Save schema from first batch */
+    if (!st->has_schema) {
+        st->schema_n_cols = in->n_cols;
+        st->schema_names = malloc(in->n_cols * sizeof(char *));
+        st->schema_types = malloc(in->n_cols * sizeof(tf_type));
+        for (size_t c = 0; c < in->n_cols; c++) {
+            st->schema_names[c] = strdup(in->col_names[c]);
+            st->schema_types[c] = in->col_types[c];
+        }
+        st->has_schema = 1;
+
+        /* Resolve column indices */
+        for (size_t i = 0; i < st->n_columns; i++) {
+            st->stats[i].col_idx = tf_batch_col_index(in, st->columns[i]);
+        }
+    }
+
+    /* Buffer rows and update stats */
+    for (size_t r = 0; r < in->n_rows; r++) {
+        add_row(st, in, r);
+
+        for (size_t i = 0; i < st->n_columns; i++) {
+            int ci = st->stats[i].col_idx;
+            if (ci < 0 || tf_batch_is_null(in, r, ci)) continue;
+
+            double val = get_numeric(in, r, ci);
+            col_stats *cs = &st->stats[i];
+            cs->count++;
+            double delta = val - cs->mean;
+            cs->mean += delta / (double)cs->count;
+            double delta2 = val - cs->mean;
+            cs->m2 += delta * delta2;
+
+            if (cs->count == 1 || val < cs->min_val) cs->min_val = val;
+            if (cs->count == 1 || val > cs->max_val) cs->max_val = val;
+        }
+    }
+
+    return TF_OK;
+}
+
+static int normalize_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)side;
+    normalize_state *st = self->state;
+    *out = NULL;
+
+    if (st->n_rows == 0) return TF_OK;
+
+    tf_batch *ob = tf_batch_create(st->schema_n_cols, st->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < st->schema_n_cols; c++) {
+        /* Normalized columns become FLOAT64 */
+        tf_type type = st->schema_types[c];
+        int is_norm_col = 0;
+        for (size_t i = 0; i < st->n_columns; i++) {
+            if (st->stats[i].col_idx == (int)c) { is_norm_col = 1; break; }
+        }
+        tf_batch_set_schema(ob, c, st->schema_names[c],
+                            is_norm_col ? TF_TYPE_FLOAT64 : type);
+    }
+
+    for (size_t r = 0; r < st->n_rows; r++) {
+        tf_batch *rb = st->rows[r].batch;
+        tf_batch_copy_row(ob, r, rb, 0);
+
+        /* Normalize target columns */
+        for (size_t i = 0; i < st->n_columns; i++) {
+            int ci = st->stats[i].col_idx;
+            if (ci < 0 || tf_batch_is_null(rb, 0, ci)) continue;
+
+            double val = get_numeric(rb, 0, ci);
+            col_stats *cs = &st->stats[i];
+            double norm;
+
+            if (st->method == NORM_MINMAX) {
+                double range = cs->max_val - cs->min_val;
+                norm = (range > 0) ? (val - cs->min_val) / range : 0;
+            } else {
+                double std = (cs->count > 1) ? sqrt(cs->m2 / (double)(cs->count - 1)) : 1;
+                norm = (std > 0) ? (val - cs->mean) / std : 0;
+            }
+            tf_batch_set_float64(ob, r, ci, norm);
+        }
+        ob->n_rows = r + 1;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static void normalize_destroy(tf_step *self) {
+    normalize_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n_columns; i++)
+            free(st->columns[i]);
+        free(st->columns);
+        free(st->stats);
+        for (size_t i = 0; i < st->n_rows; i++)
+            tf_batch_free(st->rows[i].batch);
+        free(st->rows);
+        if (st->schema_names) {
+            for (size_t c = 0; c < st->schema_n_cols; c++)
+                free(st->schema_names[c]);
+            free(st->schema_names);
+        }
+        free(st->schema_types);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_normalize_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *cols_j = cJSON_GetObjectItemCaseSensitive(args, "columns");
+    if (!cols_j || !cJSON_IsArray(cols_j)) return NULL;
+
+    int n = cJSON_GetArraySize(cols_j);
+    if (n == 0) return NULL;
+
+    normalize_state *st = calloc(1, sizeof(normalize_state));
+    if (!st) return NULL;
+
+    st->n_columns = n;
+    st->columns = malloc(n * sizeof(char *));
+    st->stats = calloc(n, sizeof(col_stats));
+    for (int i = 0; i < n; i++) {
+        cJSON *item = cJSON_GetArrayItem(cols_j, i);
+        st->columns[i] = strdup(cJSON_IsString(item) ? item->valuestring : "");
+        st->stats[i].col_idx = -1;
+    }
+
+    cJSON *method_j = cJSON_GetObjectItemCaseSensitive(args, "method");
+    st->method = parse_method(cJSON_IsString(method_j) ? method_j->valuestring : NULL);
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { normalize_destroy(&(tf_step){.state = st}); return NULL; }
+    step->process = normalize_process;
+    step->flush = normalize_flush;
+    step->destroy = normalize_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_onehot.c

@@ -0,0 +1,185 @@
+/*
+ * op_onehot.c — One-hot encoding of a categorical column.
+ * Expands a single column into N binary (0/1) columns.
+ *
+ * Config: {"column": "city", "drop": false}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char  *value;     /* category value string */
+    char  *col_name;  /* generated column name: "column_value" */
+} onehot_category;
+
+typedef struct {
+    char             *column;
+    int               drop;    /* drop original column */
+    onehot_category  *cats;
+    size_t            n_cats;
+    size_t            cap;
+} onehot_state;
+
+static const char *get_string_value(const tf_batch *b, size_t r, int ci, char *buf, size_t bufsz) {
+    if (tf_batch_is_null(b, r, ci)) return NULL;
+    switch (b->col_types[ci]) {
+        case TF_TYPE_STRING: return tf_batch_get_string(b, r, ci);
+        case TF_TYPE_INT64:
+            snprintf(buf, bufsz, "%lld", (long long)tf_batch_get_int64(b, r, ci));
+            return buf;
+        case TF_TYPE_FLOAT64:
+            snprintf(buf, bufsz, "%.17g", tf_batch_get_float64(b, r, ci));
+            return buf;
+        case TF_TYPE_BOOL:
+            return tf_batch_get_bool(b, r, ci) ? "true" : "false";
+        default: return NULL;
+    }
+}
+
+static int find_or_add_category(onehot_state *st, const char *val) {
+    for (size_t i = 0; i < st->n_cats; i++) {
+        if (strcmp(st->cats[i].value, val) == 0)
+            return (int)i;
+    }
+    /* Add new category */
+    if (st->n_cats >= st->cap) {
+        size_t newcap = st->cap ? st->cap * 2 : 16;
+        onehot_category *tmp = realloc(st->cats, newcap * sizeof(onehot_category));
+        if (!tmp) return -1;
+        st->cats = tmp;
+        st->cap = newcap;
+    }
+    st->cats[st->n_cats].value = strdup(val);
+    char namebuf[512];
+    snprintf(namebuf, sizeof(namebuf), "%s_%s", st->column, val);
+    st->cats[st->n_cats].col_name = strdup(namebuf);
+    st->n_cats++;
+    return (int)(st->n_cats - 1);
+}
+
+static int onehot_process(tf_step *self, tf_batch *in, tf_batch **out,
+                          tf_side_channels *side) {
+    (void)side;
+    onehot_state *st = self->state;
+    *out = NULL;
+
+    int ci = tf_batch_col_index(in, st->column);
+
+    /* First pass: discover any new categories in this batch */
+    size_t cats_before = st->n_cats;
+    if (ci >= 0) {
+        char buf[64];
+        for (size_t r = 0; r < in->n_rows; r++) {
+            const char *val = get_string_value(in, r, ci, buf, sizeof(buf));
+            if (val) find_or_add_category(st, val);
+        }
+    }
+    (void)cats_before;
+
+    /* Compute output column count */
+    size_t out_cols = st->drop ? (in->n_cols - 1 + st->n_cats) : (in->n_cols + st->n_cats);
+
+    tf_batch *ob = tf_batch_create(out_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+
+    /* Set schema: copy input cols (optionally skipping target), append onehot cols */
+    size_t oc = 0;
+    for (size_t c = 0; c < in->n_cols; c++) {
+        if (st->drop && ci >= 0 && c == (size_t)ci) continue;
+        tf_batch_set_schema(ob, oc, in->col_names[c], in->col_types[c]);
+        oc++;
+    }
+    for (size_t i = 0; i < st->n_cats; i++) {
+        tf_batch_set_schema(ob, oc + i, st->cats[i].col_name, TF_TYPE_INT64);
+    }
+
+    /* Fill rows */
+    char buf[64];
+    for (size_t r = 0; r < in->n_rows; r++) {
+        /* Copy input columns */
+        oc = 0;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (st->drop && ci >= 0 && c == (size_t)ci) continue;
+            if (tf_batch_is_null(in, r, c)) {
+                tf_batch_set_null(ob, r, oc);
+            } else {
+                switch (in->col_types[c]) {
+                    case TF_TYPE_STRING:
+                        tf_batch_set_string(ob, r, oc, tf_batch_get_string(in, r, c)); break;
+                    case TF_TYPE_INT64:
+                        tf_batch_set_int64(ob, r, oc, tf_batch_get_int64(in, r, c)); break;
+                    case TF_TYPE_FLOAT64:
+                        tf_batch_set_float64(ob, r, oc, tf_batch_get_float64(in, r, c)); break;
+                    case TF_TYPE_BOOL:
+                        tf_batch_set_bool(ob, r, oc, tf_batch_get_bool(in, r, c)); break;
+                    case TF_TYPE_DATE:
+                        tf_batch_set_date(ob, r, oc, tf_batch_get_date(in, r, c)); break;
+                    case TF_TYPE_TIMESTAMP:
+                        tf_batch_set_timestamp(ob, r, oc, tf_batch_get_timestamp(in, r, c)); break;
+                    default: tf_batch_set_null(ob, r, oc); break;
+                }
+            }
+            oc++;
+        }
+
+        /* Set onehot columns */
+        const char *val = (ci >= 0) ? get_string_value(in, r, ci, buf, sizeof(buf)) : NULL;
+        int match = -1;
+        if (val) {
+            for (size_t i = 0; i < st->n_cats; i++) {
+                if (strcmp(st->cats[i].value, val) == 0) { match = (int)i; break; }
+            }
+        }
+        for (size_t i = 0; i < st->n_cats; i++) {
+            tf_batch_set_int64(ob, r, oc + i, (int)i == match ? 1 : 0);
+        }
+
+        ob->n_rows = r + 1;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int onehot_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void onehot_destroy(tf_step *self) {
+    onehot_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n_cats; i++) {
+            free(st->cats[i].value);
+            free(st->cats[i].col_name);
+        }
+        free(st->cats);
+        free(st->column);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_onehot_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    if (!cJSON_IsString(col_j)) return NULL;
+
+    onehot_state *st = calloc(1, sizeof(onehot_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+
+    cJSON *drop_j = cJSON_GetObjectItemCaseSensitive(args, "drop");
+    st->drop = cJSON_IsBool(drop_j) && cJSON_IsTrue(drop_j) ? 1 : 0;
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st->column); free(st); return NULL; }
+    step->process = onehot_process;
+    step->flush = onehot_flush;
+    step->destroy = onehot_destroy;
+    step->state = st;
+    return step;
+}