tranfi 0.0.1 → 0.1.2

package/csrc/op_hash.c

@@ -0,0 +1,126 @@
+/*
+ * op_hash.c — DJB2 hash of columns, adds _hash column.
+ *
+ * Config: {"columns": ["name", "city"]}
+ *   or {} for all columns.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include "date_utils.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char  **cols;
+    size_t  n_cols;
+} hash_state;
+
+static int hash_process(tf_step *self, tf_batch *in, tf_batch **out,
+                        tf_side_channels *side) {
+    (void)side;
+    hash_state *st = self->state;
+    *out = NULL;
+
+    tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    tf_batch_set_schema(ob, in->n_cols, "_hash", TF_TYPE_INT64);
+
+    /* Resolve column indices */
+    size_t n_keys;
+    int *col_indices;
+    if (st->n_cols > 0) {
+        n_keys = st->n_cols;
+        col_indices = malloc(n_keys * sizeof(int));
+        if (!col_indices) { tf_batch_free(ob); return TF_ERROR; }
+        for (size_t k = 0; k < n_keys; k++)
+            col_indices[k] = tf_batch_col_index(in, st->cols[k]);
+    } else {
+        n_keys = in->n_cols;
+        col_indices = malloc(n_keys * sizeof(int));
+        if (!col_indices) { tf_batch_free(ob); return TF_ERROR; }
+        for (size_t k = 0; k < n_keys; k++) col_indices[k] = (int)k;
+    }
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        tf_batch_copy_row(ob, r, in, r);
+        /* Compute hash */
+        uint32_t h = 5381;
+        char val_buf[64];
+        for (size_t k = 0; k < n_keys; k++) {
+            int ci = col_indices[k];
+            if (ci < 0 || tf_batch_is_null(in, r, ci)) continue;
+            const char *val;
+            switch (in->col_types[ci]) {
+                case TF_TYPE_STRING: val = tf_batch_get_string(in, r, ci); break;
+                case TF_TYPE_INT64:
+                    snprintf(val_buf, sizeof(val_buf), "%lld", (long long)tf_batch_get_int64(in, r, ci));
+                    val = val_buf; break;
+                case TF_TYPE_FLOAT64:
+                    snprintf(val_buf, sizeof(val_buf), "%.17g", tf_batch_get_float64(in, r, ci));
+                    val = val_buf; break;
+                case TF_TYPE_BOOL:
+                    val = tf_batch_get_bool(in, r, ci) ? "T" : "F"; break;
+                case TF_TYPE_DATE:
+                    snprintf(val_buf, sizeof(val_buf), "%d", (int)tf_batch_get_date(in, r, ci));
+                    val = val_buf; break;
+                case TF_TYPE_TIMESTAMP:
+                    snprintf(val_buf, sizeof(val_buf), "%lld", (long long)tf_batch_get_timestamp(in, r, ci));
+                    val = val_buf; break;
+                default: val = ""; break;
+            }
+            for (const unsigned char *p = (const unsigned char *)val; *p; p++)
+                h = ((h << 5) + h) ^ *p;
+        }
+        tf_batch_set_int64(ob, r, in->n_cols, (int64_t)h);
+        ob->n_rows = r + 1;
+    }
+
+    free(col_indices);
+    *out = ob;
+    return TF_OK;
+}
+
+static int hash_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void hash_destroy(tf_step *self) {
+    hash_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i]);
+        free(st->cols); free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_hash_create(const cJSON *args) {
+    hash_state *st = calloc(1, sizeof(hash_state));
+    if (!st) return NULL;
+
+    if (args) {
+        cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
+        if (columns && cJSON_IsArray(columns)) {
+            int n = cJSON_GetArraySize(columns);
+            if (n > 0) {
+                st->cols = calloc(n, sizeof(char *));
+                st->n_cols = n;
+                for (int i = 0; i < n; i++) {
+                    cJSON *item = cJSON_GetArrayItem(columns, i);
+                    if (cJSON_IsString(item)) st->cols[i] = strdup(item->valuestring);
+                }
+            }
+        }
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { hash_destroy(&(tf_step){.state = st}); return NULL; }
+    step->process = hash_process;
+    step->flush = hash_flush;
+    step->destroy = hash_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_head.c

@@ -0,0 +1,149 @@
+/*
+ * op_head.c — Take first N rows.
+ *
+ * Config: {"n": 5}
+ * Passes through rows until N have been seen, then emits empty batches.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    size_t limit;
+    size_t seen;
+} head_state;
+
+static int head_process(tf_step *self, tf_batch *in, tf_batch **out,
+                        tf_side_channels *side) {
+    (void)side;
+    head_state *st = self->state;
+    *out = NULL;
+
+    if (st->seen >= st->limit) {
+        /* Already past limit, discard */
+        return TF_OK;
+    }
+
+    size_t remaining = st->limit - st->seen;
+    size_t take = in->n_rows < remaining ? in->n_rows : remaining;
+
+    if (take == in->n_rows) {
+        /* Take all rows — create a copy */
+        tf_batch *ob = tf_batch_create(in->n_cols, take);
+        if (!ob) return TF_ERROR;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+        }
+        for (size_t r = 0; r < take; r++) {
+            tf_batch_ensure_capacity(ob, r + 1);
+            for (size_t c = 0; c < in->n_cols; c++) {
+                if (tf_batch_is_null(in, r, c)) {
+                    tf_batch_set_null(ob, r, c);
+                    continue;
+                }
+                switch (in->col_types[c]) {
+                    case TF_TYPE_BOOL:
+                        tf_batch_set_bool(ob, r, c, tf_batch_get_bool(in, r, c));
+                        break;
+                    case TF_TYPE_INT64:
+                        tf_batch_set_int64(ob, r, c, tf_batch_get_int64(in, r, c));
+                        break;
+                    case TF_TYPE_FLOAT64:
+                        tf_batch_set_float64(ob, r, c, tf_batch_get_float64(in, r, c));
+                        break;
+                    case TF_TYPE_STRING:
+                        tf_batch_set_string(ob, r, c, tf_batch_get_string(in, r, c));
+                        break;
+                    case TF_TYPE_DATE:
+                        tf_batch_set_date(ob, r, c, tf_batch_get_date(in, r, c));
+                        break;
+                    case TF_TYPE_TIMESTAMP:
+                        tf_batch_set_timestamp(ob, r, c, tf_batch_get_timestamp(in, r, c));
+                        break;
+                    default:
+                        tf_batch_set_null(ob, r, c);
+                        break;
+                }
+            }
+            ob->n_rows = r + 1;
+        }
+        st->seen += take;
+        *out = ob;
+    } else {
+        /* Partial take — only copy first `take` rows */
+        tf_batch *ob = tf_batch_create(in->n_cols, take);
+        if (!ob) return TF_ERROR;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+        }
+        for (size_t r = 0; r < take; r++) {
+            tf_batch_ensure_capacity(ob, r + 1);
+            for (size_t c = 0; c < in->n_cols; c++) {
+                if (tf_batch_is_null(in, r, c)) {
+                    tf_batch_set_null(ob, r, c);
+                    continue;
+                }
+                switch (in->col_types[c]) {
+                    case TF_TYPE_BOOL:
+                        tf_batch_set_bool(ob, r, c, tf_batch_get_bool(in, r, c));
+                        break;
+                    case TF_TYPE_INT64:
+                        tf_batch_set_int64(ob, r, c, tf_batch_get_int64(in, r, c));
+                        break;
+                    case TF_TYPE_FLOAT64:
+                        tf_batch_set_float64(ob, r, c, tf_batch_get_float64(in, r, c));
+                        break;
+                    case TF_TYPE_STRING:
+                        tf_batch_set_string(ob, r, c, tf_batch_get_string(in, r, c));
+                        break;
+                    case TF_TYPE_DATE:
+                        tf_batch_set_date(ob, r, c, tf_batch_get_date(in, r, c));
+                        break;
+                    case TF_TYPE_TIMESTAMP:
+                        tf_batch_set_timestamp(ob, r, c, tf_batch_get_timestamp(in, r, c));
+                        break;
+                    default:
+                        tf_batch_set_null(ob, r, c);
+                        break;
+                }
+            }
+            ob->n_rows = r + 1;
+        }
+        st->seen += take;
+        *out = ob;
+    }
+
+    return TF_OK;
+}
+
+static int head_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side;
+    *out = NULL;
+    return TF_OK;
+}
+
+static void head_destroy(tf_step *self) {
+    free(self->state);
+    free(self);
+}
+
+tf_step *tf_head_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *n_json = cJSON_GetObjectItemCaseSensitive(args, "n");
+    if (!cJSON_IsNumber(n_json) || n_json->valueint <= 0) return NULL;
+
+    head_state *st = calloc(1, sizeof(head_state));
+    if (!st) return NULL;
+    st->limit = (size_t)n_json->valueint;
+    st->seen = 0;
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st); return NULL; }
+    step->process = head_process;
+    step->flush = head_flush;
+    step->destroy = head_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_interpolate.c

@@ -0,0 +1,239 @@
+/*
+ * op_interpolate.c — Fill null values via interpolation.
+ * Methods: forward, backward, linear.
+ *
+ * For backward/linear: buffers rows with null target values and emits
+ * them when the next non-null value arrives.
+ *
+ * Config: {"column": "price", "method": "linear"}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef enum {
+    INTERP_FORWARD,
+    INTERP_BACKWARD,
+    INTERP_LINEAR
+} interp_method;
+
+/* Buffered row: we store complete batches and track which rows are pending */
+typedef struct pending_row {
+    tf_batch *batch; /* single-row batch (copy of original row) */
+    size_t    target_col; /* column index in the batch */
+} pending_row;
+
+typedef struct {
+    char          *column;
+    interp_method  method;
+    double         last_val;
+    int            has_last;
+    /* Pending null rows (for backward/linear) */
+    pending_row   *pending;
+    size_t         n_pending;
+    size_t         cap_pending;
+} interpolate_state;
+
+static interp_method parse_method(const char *s) {
+    if (!s) return INTERP_LINEAR;
+    if (strcmp(s, "forward") == 0) return INTERP_FORWARD;
+    if (strcmp(s, "backward") == 0) return INTERP_BACKWARD;
+    return INTERP_LINEAR;
+}
+
+static double get_numeric(const tf_batch *b, size_t r, int ci) {
+    if (b->col_types[ci] == TF_TYPE_INT64) return (double)tf_batch_get_int64(b, r, ci);
+    if (b->col_types[ci] == TF_TYPE_FLOAT64) return tf_batch_get_float64(b, r, ci);
+    return 0;
+}
+
+static void add_pending(interpolate_state *st, tf_batch *row_batch, size_t target_col) {
+    if (st->n_pending >= st->cap_pending) {
+        size_t newcap = st->cap_pending ? st->cap_pending * 2 : 16;
+        pending_row *tmp = realloc(st->pending, newcap * sizeof(pending_row));
+        if (!tmp) return;
+        st->pending = tmp;
+        st->cap_pending = newcap;
+    }
+    st->pending[st->n_pending].batch = row_batch;
+    st->pending[st->n_pending].target_col = target_col;
+    st->n_pending++;
+}
+
+/* Emit all pending rows into output batch, interpolating values */
+static void flush_pending(interpolate_state *st, tf_batch *ob, size_t *out_row,
+                          double end_val, size_t target_col) {
+    if (st->n_pending == 0) return;
+
+    for (size_t i = 0; i < st->n_pending; i++) {
+        tf_batch *pb = st->pending[i].batch;
+        size_t r = *out_row;
+        tf_batch_copy_row(ob, r, pb, 0);
+
+        double interp_val;
+        if (st->method == INTERP_BACKWARD) {
+            interp_val = end_val;
+        } else {
+            /* Linear: interpolate between last_val and end_val */
+            if (st->has_last) {
+                double t = (double)(i + 1) / (double)(st->n_pending + 1);
+                interp_val = st->last_val + t * (end_val - st->last_val);
+            } else {
+                interp_val = end_val;
+            }
+        }
+
+        tf_batch_set_float64(ob, r, target_col, interp_val);
+        ob->n_rows = r + 1;
+        (*out_row)++;
+
+        tf_batch_free(pb);
+    }
+    st->n_pending = 0;
+}
+
+static int interpolate_process(tf_step *self, tf_batch *in, tf_batch **out,
+                               tf_side_channels *side) {
+    (void)side;
+    interpolate_state *st = self->state;
+    *out = NULL;
+
+    int ci = tf_batch_col_index(in, st->column);
+
+    /* Count total rows we might output (pending + current batch) */
+    size_t max_rows = st->n_pending + in->n_rows;
+    tf_batch *ob = tf_batch_create(in->n_cols, max_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+
+    size_t out_row = 0;
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (ci < 0) {
+            /* No target column — just pass through */
+            tf_batch_copy_row(ob, out_row, in, r);
+            ob->n_rows = out_row + 1;
+            out_row++;
+            continue;
+        }
+
+        int is_null = tf_batch_is_null(in, r, ci);
+
+        if (is_null) {
+            if (st->method == INTERP_FORWARD && st->has_last) {
+                /* Forward fill: use last known value */
+                tf_batch_copy_row(ob, out_row, in, r);
+                tf_batch_set_float64(ob, out_row, ci, st->last_val);
+                ob->n_rows = out_row + 1;
+                out_row++;
+            } else if (st->method == INTERP_FORWARD) {
+                /* No previous value yet — pass null through */
+                tf_batch_copy_row(ob, out_row, in, r);
+                ob->n_rows = out_row + 1;
+                out_row++;
+            } else {
+                /* Backward/linear: buffer this row */
+                tf_batch *row_copy = tf_batch_create(in->n_cols, 1);
+                if (row_copy) {
+                    for (size_t c = 0; c < in->n_cols; c++)
+                        tf_batch_set_schema(row_copy, c, in->col_names[c], in->col_types[c]);
+                    tf_batch_copy_row(row_copy, 0, in, r);
+                    row_copy->n_rows = 1;
+                    add_pending(st, row_copy, ci);
+                }
+            }
+        } else {
+            double val = get_numeric(in, r, ci);
+
+            /* Flush any pending rows */
+            if (st->n_pending > 0) {
+                /* Grow output if needed */
+                tf_batch_ensure_capacity(ob, out_row + st->n_pending + 1);
+                flush_pending(st, ob, &out_row, val, ci);
+            }
+
+            /* Output current row */
+            tf_batch_copy_row(ob, out_row, in, r);
+            ob->n_rows = out_row + 1;
+            out_row++;
+
+            st->last_val = val;
+            st->has_last = 1;
+        }
+    }
+
+    if (ob->n_rows == 0) {
+        tf_batch_free(ob);
+        *out = NULL;
+    } else {
+        *out = ob;
+    }
+    return TF_OK;
+}
+
+static int interpolate_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)side;
+    interpolate_state *st = self->state;
+    *out = NULL;
+
+    /* Emit any remaining pending rows as nulls (or forward-fill with last known) */
+    if (st->n_pending > 0) {
+        tf_batch *first = st->pending[0].batch;
+        tf_batch *ob = tf_batch_create(first->n_cols, st->n_pending);
+        if (!ob) return TF_OK;
+        for (size_t c = 0; c < first->n_cols; c++)
+            tf_batch_set_schema(ob, c, first->col_names[c], first->col_types[c]);
+
+        for (size_t i = 0; i < st->n_pending; i++) {
+            tf_batch *pb = st->pending[i].batch;
+            tf_batch_copy_row(ob, i, pb, 0);
+            /* For linear/backward at end of stream: use last known if available */
+            if (st->has_last) {
+                tf_batch_set_float64(ob, i, st->pending[i].target_col, st->last_val);
+            }
+            ob->n_rows = i + 1;
+            tf_batch_free(pb);
+        }
+        st->n_pending = 0;
+        *out = ob;
+    }
+
+    return TF_OK;
+}
+
+static void interpolate_destroy(tf_step *self) {
+    interpolate_state *st = self->state;
+    if (st) {
+        for (size_t i = 0; i < st->n_pending; i++)
+            tf_batch_free(st->pending[i].batch);
+        free(st->pending);
+        free(st->column);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_interpolate_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    if (!cJSON_IsString(col_j)) return NULL;
+
+    interpolate_state *st = calloc(1, sizeof(interpolate_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+
+    cJSON *method_j = cJSON_GetObjectItemCaseSensitive(args, "method");
+    st->method = parse_method(cJSON_IsString(method_j) ? method_j->valuestring : NULL);
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st->column); free(st); return NULL; }
+    step->process = interpolate_process;
+    step->flush = interpolate_flush;
+    step->destroy = interpolate_destroy;
+    step->state = st;
+    return step;
+}