tranfi 0.0.1 → 0.1.2

package/csrc/op_skip.c

@@ -0,0 +1,152 @@
+/*
+ * op_skip.c — Skip first N rows.
+ *
+ * Config: {"n": 5}
+ * Discards the first N rows, passes through the rest.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    size_t n;
+    size_t seen;
+} skip_state;
+
+static int skip_process(tf_step *self, tf_batch *in, tf_batch **out,
+                        tf_side_channels *side) {
+    (void)side;
+    skip_state *st = self->state;
+    *out = NULL;
+
+    if (st->seen >= st->n) {
+        /* Already past skip region — pass through all rows */
+        tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+        if (!ob) return TF_ERROR;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+        }
+        for (size_t r = 0; r < in->n_rows; r++) {
+            tf_batch_ensure_capacity(ob, r + 1);
+            for (size_t c = 0; c < in->n_cols; c++) {
+                if (tf_batch_is_null(in, r, c)) {
+                    tf_batch_set_null(ob, r, c);
+                    continue;
+                }
+                switch (in->col_types[c]) {
+                    case TF_TYPE_BOOL:
+                        tf_batch_set_bool(ob, r, c, tf_batch_get_bool(in, r, c));
+                        break;
+                    case TF_TYPE_INT64:
+                        tf_batch_set_int64(ob, r, c, tf_batch_get_int64(in, r, c));
+                        break;
+                    case TF_TYPE_FLOAT64:
+                        tf_batch_set_float64(ob, r, c, tf_batch_get_float64(in, r, c));
+                        break;
+                    case TF_TYPE_STRING:
+                        tf_batch_set_string(ob, r, c, tf_batch_get_string(in, r, c));
+                        break;
+                    case TF_TYPE_DATE:
+                        tf_batch_set_date(ob, r, c, tf_batch_get_date(in, r, c));
+                        break;
+                    case TF_TYPE_TIMESTAMP:
+                        tf_batch_set_timestamp(ob, r, c, tf_batch_get_timestamp(in, r, c));
+                        break;
+                    default:
+                        tf_batch_set_null(ob, r, c);
+                        break;
+                }
+            }
+            ob->n_rows = r + 1;
+        }
+        *out = ob;
+        return TF_OK;
+    }
+
+    size_t remaining_skip = st->n - st->seen;
+
+    if (remaining_skip >= in->n_rows) {
+        /* Skip entire batch */
+        st->seen += in->n_rows;
+        return TF_OK;
+    }
+
+    /* Partial skip — emit rows after the skip region */
+    size_t emit_start = remaining_skip;
+    size_t emit_count = in->n_rows - emit_start;
+    st->seen = st->n;
+
+    tf_batch *ob = tf_batch_create(in->n_cols, emit_count);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++) {
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    }
+    for (size_t i = 0; i < emit_count; i++) {
+        size_t r = emit_start + i;
+        tf_batch_ensure_capacity(ob, i + 1);
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (tf_batch_is_null(in, r, c)) {
+                tf_batch_set_null(ob, i, c);
+                continue;
+            }
+            switch (in->col_types[c]) {
+                case TF_TYPE_BOOL:
+                    tf_batch_set_bool(ob, i, c, tf_batch_get_bool(in, r, c));
+                    break;
+                case TF_TYPE_INT64:
+                    tf_batch_set_int64(ob, i, c, tf_batch_get_int64(in, r, c));
+                    break;
+                case TF_TYPE_FLOAT64:
+                    tf_batch_set_float64(ob, i, c, tf_batch_get_float64(in, r, c));
+                    break;
+                case TF_TYPE_STRING:
+                    tf_batch_set_string(ob, i, c, tf_batch_get_string(in, r, c));
+                    break;
+                case TF_TYPE_DATE:
+                    tf_batch_set_date(ob, i, c, tf_batch_get_date(in, r, c));
+                    break;
+                case TF_TYPE_TIMESTAMP:
+                    tf_batch_set_timestamp(ob, i, c, tf_batch_get_timestamp(in, r, c));
+                    break;
+                default:
+                    tf_batch_set_null(ob, i, c);
+                    break;
+            }
+        }
+        ob->n_rows = i + 1;
+    }
+    *out = ob;
+    return TF_OK;
+}
+
+static int skip_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side;
+    *out = NULL;
+    return TF_OK;
+}
+
+static void skip_destroy(tf_step *self) {
+    free(self->state);
+    free(self);
+}
+
+tf_step *tf_skip_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *n_json = cJSON_GetObjectItemCaseSensitive(args, "n");
+    if (!cJSON_IsNumber(n_json) || n_json->valueint <= 0) return NULL;
+
+    skip_state *st = calloc(1, sizeof(skip_state));
+    if (!st) return NULL;
+    st->n = (size_t)n_json->valueint;
+    st->seen = 0;
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st); return NULL; }
+    step->process = skip_process;
+    step->flush = skip_flush;
+    step->destroy = skip_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_sort.c

@@ -0,0 +1,273 @@
+/*
+ * op_sort.c — Sort all rows by column(s). Requires buffering all data.
+ *
+ * Config: {"columns": [{"name": "age", "desc": false}, {"name": "name", "desc": true}]}
+ * Buffers all incoming batches, sorts on flush, emits as a single batch.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char *name;
+    int   desc;   /* 1 = descending */
+} sort_col;
+
+typedef struct {
+    /* Accumulated data */
+    tf_batch *buf;       /* growing buffer of all rows */
+    int       has_schema;
+
+    /* Sort spec */
+    sort_col *cols;
+    size_t    n_cols;
+} sort_state;
+
+/* Copy a row from src batch to dst batch */
+static int copy_row(tf_batch *dst, size_t dst_row,
+                    const tf_batch *src, size_t src_row) {
+    if (tf_batch_ensure_capacity(dst, dst_row + 1) != TF_OK) return TF_ERROR;
+    for (size_t c = 0; c < src->n_cols; c++) {
+        if (tf_batch_is_null(src, src_row, c)) {
+            tf_batch_set_null(dst, dst_row, c);
+            continue;
+        }
+        switch (src->col_types[c]) {
+            case TF_TYPE_BOOL:
+                tf_batch_set_bool(dst, dst_row, c, tf_batch_get_bool(src, src_row, c));
+                break;
+            case TF_TYPE_INT64:
+                tf_batch_set_int64(dst, dst_row, c, tf_batch_get_int64(src, src_row, c));
+                break;
+            case TF_TYPE_FLOAT64:
+                tf_batch_set_float64(dst, dst_row, c, tf_batch_get_float64(src, src_row, c));
+                break;
+            case TF_TYPE_STRING:
+                tf_batch_set_string(dst, dst_row, c, tf_batch_get_string(src, src_row, c));
+                break;
+            case TF_TYPE_DATE:
+                tf_batch_set_date(dst, dst_row, c, tf_batch_get_date(src, src_row, c));
+                break;
+            case TF_TYPE_TIMESTAMP:
+                tf_batch_set_timestamp(dst, dst_row, c, tf_batch_get_timestamp(src, src_row, c));
+                break;
+            default:
+                tf_batch_set_null(dst, dst_row, c);
+                break;
+        }
+    }
+    return TF_OK;
+}
+
+static int sort_process(tf_step *self, tf_batch *in, tf_batch **out,
+                        tf_side_channels *side) {
+    (void)side;
+    sort_state *st = self->state;
+    *out = NULL;
+
+    /* Initialize buffer on first batch */
+    if (!st->has_schema) {
+        st->buf = tf_batch_create(in->n_cols, in->n_rows > 0 ? in->n_rows : 16);
+        if (!st->buf) return TF_ERROR;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            tf_batch_set_schema(st->buf, c, in->col_names[c], in->col_types[c]);
+        }
+        st->has_schema = 1;
+    }
+
+    /* Append all rows */
+    for (size_t r = 0; r < in->n_rows; r++) {
+        size_t dst_row = st->buf->n_rows;
+        if (copy_row(st->buf, dst_row, in, r) != TF_OK) return TF_ERROR;
+        st->buf->n_rows = dst_row + 1;
+    }
+
+    return TF_OK;
+}
+
+/* Comparator context for qsort_r / qsort */
+typedef struct {
+    const tf_batch *batch;
+    int            *col_indices;
+    int            *col_desc;
+    size_t          n_sort_cols;
+} sort_ctx;
+
+static sort_ctx *g_sort_ctx; /* global for qsort (no qsort_r on all platforms) */
+
+static int compare_rows(const void *a, const void *b) {
+    size_t ra = *(const size_t *)a;
+    size_t rb = *(const size_t *)b;
+    const tf_batch *batch = g_sort_ctx->batch;
+
+    for (size_t k = 0; k < g_sort_ctx->n_sort_cols; k++) {
+        int ci = g_sort_ctx->col_indices[k];
+        if (ci < 0) continue;
+
+        int null_a = tf_batch_is_null(batch, ra, ci);
+        int null_b = tf_batch_is_null(batch, rb, ci);
+
+        /* Nulls sort last */
+        if (null_a && null_b) continue;
+        if (null_a) return 1;
+        if (null_b) return -1;
+
+        int cmp = 0;
+        switch (batch->col_types[ci]) {
+            case TF_TYPE_INT64: {
+                int64_t va = tf_batch_get_int64(batch, ra, ci);
+                int64_t vb = tf_batch_get_int64(batch, rb, ci);
+                cmp = (va > vb) - (va < vb);
+                break;
+            }
+            case TF_TYPE_FLOAT64: {
+                double va = tf_batch_get_float64(batch, ra, ci);
+                double vb = tf_batch_get_float64(batch, rb, ci);
+                cmp = (va > vb) - (va < vb);
+                break;
+            }
+            case TF_TYPE_STRING:
+                cmp = strcmp(tf_batch_get_string(batch, ra, ci),
+                             tf_batch_get_string(batch, rb, ci));
+                break;
+            case TF_TYPE_BOOL: {
+                bool va = tf_batch_get_bool(batch, ra, ci);
+                bool vb = tf_batch_get_bool(batch, rb, ci);
+                cmp = (int)va - (int)vb;
+                break;
+            }
+            case TF_TYPE_DATE: {
+                int32_t va = tf_batch_get_date(batch, ra, ci);
+                int32_t vb = tf_batch_get_date(batch, rb, ci);
+                cmp = (va > vb) - (va < vb);
+                break;
+            }
+            case TF_TYPE_TIMESTAMP: {
+                int64_t va = tf_batch_get_timestamp(batch, ra, ci);
+                int64_t vb = tf_batch_get_timestamp(batch, rb, ci);
+                cmp = (va > vb) - (va < vb);
+                break;
+            }
+            default:
+                break;
+        }
+
+        if (cmp != 0) {
+            return g_sort_ctx->col_desc[k] ? -cmp : cmp;
+        }
+    }
+    return 0;
+}
+
+static int sort_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)side;
+    sort_state *st = self->state;
+    *out = NULL;
+
+    if (!st->buf || st->buf->n_rows == 0) return TF_OK;
+
+    size_t n = st->buf->n_rows;
+
+    /* Resolve sort column indices */
+    int *col_indices = malloc(st->n_cols * sizeof(int));
+    int *col_desc = malloc(st->n_cols * sizeof(int));
+    if (!col_indices || !col_desc) { free(col_indices); free(col_desc); return TF_ERROR; }
+
+    for (size_t k = 0; k < st->n_cols; k++) {
+        col_indices[k] = tf_batch_col_index(st->buf, st->cols[k].name);
+        col_desc[k] = st->cols[k].desc;
+    }
+
+    /* Build index array */
+    size_t *indices = malloc(n * sizeof(size_t));
+    if (!indices) { free(col_indices); free(col_desc); return TF_ERROR; }
+    for (size_t i = 0; i < n; i++) indices[i] = i;
+
+    /* Sort */
+    sort_ctx ctx = {
+        .batch = st->buf,
+        .col_indices = col_indices,
+        .col_desc = col_desc,
+        .n_sort_cols = st->n_cols,
+    };
+    g_sort_ctx = &ctx;
+    qsort(indices, n, sizeof(size_t), compare_rows);
+    g_sort_ctx = NULL;
+
+    /* Build output batch in sorted order */
+    tf_batch *ob = tf_batch_create(st->buf->n_cols, n);
+    if (!ob) { free(indices); free(col_indices); free(col_desc); return TF_ERROR; }
+    for (size_t c = 0; c < st->buf->n_cols; c++) {
+        tf_batch_set_schema(ob, c, st->buf->col_names[c], st->buf->col_types[c]);
+    }
+
+    for (size_t i = 0; i < n; i++) {
+        if (copy_row(ob, i, st->buf, indices[i]) != TF_OK) {
+            free(indices); free(col_indices); free(col_desc);
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        ob->n_rows = i + 1;
+    }
+
+    free(indices);
+    free(col_indices);
+    free(col_desc);
+
+    *out = ob;
+    return TF_OK;
+}
+
+static void sort_destroy(tf_step *self) {
+    sort_state *st = self->state;
+    if (st) {
+        if (st->buf) tf_batch_free(st->buf);
+        for (size_t i = 0; i < st->n_cols; i++) free(st->cols[i].name);
+        free(st->cols);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_sort_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *columns = cJSON_GetObjectItemCaseSensitive(args, "columns");
+    if (!columns || !cJSON_IsArray(columns)) return NULL;
+
+    int n = cJSON_GetArraySize(columns);
+    if (n <= 0) return NULL;
+
+    sort_state *st = calloc(1, sizeof(sort_state));
+    if (!st) return NULL;
+    st->cols = calloc(n, sizeof(sort_col));
+    if (!st->cols) { free(st); return NULL; }
+    st->n_cols = n;
+
+    for (int i = 0; i < n; i++) {
+        cJSON *item = cJSON_GetArrayItem(columns, i);
+        cJSON *name_j = cJSON_GetObjectItemCaseSensitive(item, "name");
+        cJSON *desc_j = cJSON_GetObjectItemCaseSensitive(item, "desc");
+        if (!cJSON_IsString(name_j)) {
+            for (int j = 0; j < i; j++) free(st->cols[j].name);
+            free(st->cols); free(st);
+            return NULL;
+        }
+        st->cols[i].name = strdup(name_j->valuestring);
+        st->cols[i].desc = (desc_j && cJSON_IsBool(desc_j)) ? cJSON_IsTrue(desc_j) : 0;
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) {
+        for (int i = 0; i < n; i++) free(st->cols[i].name);
+        free(st->cols); free(st);
+        return NULL;
+    }
+    step->process = sort_process;
+    step->flush = sort_flush;
+    step->destroy = sort_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_split.c

@@ -0,0 +1,114 @@
+/*
+ * op_split.c — Split column into multiple columns.
+ *
+ * Config: {"column": "name", "delimiter": " ", "names": ["first", "last"]}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    char  *column;
+    char  *delimiter;
+    char **names;
+    size_t n_names;
+} split_state;
+
+static int split_process(tf_step *self, tf_batch *in, tf_batch **out,
+                         tf_side_channels *side) {
+    (void)side;
+    split_state *st = self->state;
+    *out = NULL;
+
+    size_t out_cols = in->n_cols + st->n_names;
+    tf_batch *ob = tf_batch_create(out_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+
+    /* Copy existing schema */
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    /* Add new columns */
+    for (size_t k = 0; k < st->n_names; k++)
+        tf_batch_set_schema(ob, in->n_cols + k, st->names[k], TF_TYPE_STRING);
+
+    int ci = tf_batch_col_index(in, st->column);
+    size_t delim_len = strlen(st->delimiter);
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        tf_batch_copy_row(ob, r, in, r);
+        /* Initialize new columns as null */
+        for (size_t k = 0; k < st->n_names; k++)
+            tf_batch_set_null(ob, r, in->n_cols + k);
+
+        if (ci >= 0 && !tf_batch_is_null(in, r, ci) && in->col_types[ci] == TF_TYPE_STRING) {
+            const char *val = tf_batch_get_string(in, r, ci);
+            const char *p = val;
+            for (size_t k = 0; k < st->n_names && *p; k++) {
+                const char *found = strstr(p, st->delimiter);
+                size_t tok_len = found ? (size_t)(found - p) : strlen(p);
+                char *tok = malloc(tok_len + 1);
+                if (tok) {
+                    memcpy(tok, p, tok_len);
+                    tok[tok_len] = '\0';
+                    tf_batch_set_string(ob, r, in->n_cols + k, tok);
+                    free(tok);
+                }
+                if (found) p = found + delim_len;
+                else break;
+            }
+        }
+        ob->n_rows = r + 1;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int split_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void split_destroy(tf_step *self) {
+    split_state *st = self->state;
+    if (st) {
+        free(st->column); free(st->delimiter);
+        for (size_t i = 0; i < st->n_names; i++) free(st->names[i]);
+        free(st->names);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_split_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    cJSON *names_j = cJSON_GetObjectItemCaseSensitive(args, "names");
+    if (!cJSON_IsString(col_j) || !cJSON_IsArray(names_j)) return NULL;
+
+    int n = cJSON_GetArraySize(names_j);
+    if (n <= 0) return NULL;
+
+    split_state *st = calloc(1, sizeof(split_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+
+    cJSON *delim_j = cJSON_GetObjectItemCaseSensitive(args, "delimiter");
+    st->delimiter = strdup(cJSON_IsString(delim_j) ? delim_j->valuestring : " ");
+
+    st->names = calloc(n, sizeof(char *));
+    st->n_names = n;
+    for (int i = 0; i < n; i++) {
+        cJSON *item = cJSON_GetArrayItem(names_j, i);
+        if (cJSON_IsString(item)) st->names[i] = strdup(item->valuestring);
+    }
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { split_destroy(&(tf_step){.state = st}); return NULL; }
+    step->process = split_process;
+    step->flush = split_flush;
+    step->destroy = split_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_split_data.c

@@ -0,0 +1,87 @@
+/*
+ * op_split_data.c — Train/test split with probabilistic assignment.
+ * Uses LCG PRNG for reproducible splitting.
+ *
+ * Config: {"ratio": 0.8, "result": "_split", "seed": 42}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char   *result;
+    double  ratio;
+    uint64_t seed;
+    uint64_t row_index;
+} split_data_state;
+
+/* Simple LCG for deterministic random */
+static double lcg_random(uint64_t seed, uint64_t index) {
+    uint64_t x = seed ^ (index * 6364136223846793005ULL + 1442695040888963407ULL);
+    x = x * 6364136223846793005ULL + 1442695040888963407ULL;
+    x = x * 6364136223846793005ULL + 1442695040888963407ULL;
+    return (double)(x >> 33) / (double)(1ULL << 31);
+}
+
+static int split_data_process(tf_step *self, tf_batch *in, tf_batch **out,
+                              tf_side_channels *side) {
+    (void)side;
+    split_data_state *st = self->state;
+    *out = NULL;
+
+    tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
+    if (!ob) return TF_ERROR;
+    for (size_t c = 0; c < in->n_cols; c++)
+        tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]);
+    tf_batch_set_schema(ob, in->n_cols, st->result, TF_TYPE_STRING);
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        tf_batch_copy_row(ob, r, in, r);
+
+        double rval = lcg_random(st->seed, st->row_index);
+        const char *label = (rval < st->ratio) ? "train" : "test";
+        tf_batch_set_string(ob, r, in->n_cols, label);
+        ob->n_rows = r + 1;
+        st->row_index++;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int split_data_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void split_data_destroy(tf_step *self) {
+    split_data_state *st = self->state;
+    if (st) { free(st->result); free(st); }
+    free(self);
+}
+
+tf_step *tf_split_data_create(const cJSON *args) {
+    if (!args) return NULL;
+
+    split_data_state *st = calloc(1, sizeof(split_data_state));
+    if (!st) return NULL;
+
+    cJSON *ratio_j = cJSON_GetObjectItemCaseSensitive(args, "ratio");
+    st->ratio = cJSON_IsNumber(ratio_j) ? ratio_j->valuedouble : 0.8;
+
+    cJSON *seed_j = cJSON_GetObjectItemCaseSensitive(args, "seed");
+    st->seed = cJSON_IsNumber(seed_j) ? (uint64_t)seed_j->valueint : 42;
+
+    cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
+    st->result = strdup(cJSON_IsString(res_j) ? res_j->valuestring : "_split");
+
+    tf_step *step = malloc(sizeof(tf_step));
+    if (!step) { free(st->result); free(st); return NULL; }
+    step->process = split_data_process;
+    step->flush = split_data_flush;
+    step->destroy = split_data_destroy;
+    step->state = st;
+    return step;
+}