tranfi 0.0.2 → 0.2.0

package/csrc/op_source_name.c

@@ -0,0 +1,120 @@
+/*
+ * op_source_name.c -- Append the current host-provided source name.
+ *
+ * The source name is pipeline metadata set by the embedding host before push()
+ * or input-boundary flushes. The op itself is row-local and never opens files.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char *result;
+    char *default_value;
+} source_name_state;
+
+static int source_name_process(tf_step *self, tf_batch *in, tf_batch **out,
+                               tf_side_channels *side) {
+    source_name_state *st = self ? self->state : NULL;
+    if (!st || !in || !out) return TF_ERROR;
+    *out = NULL;
+
+    if (tf_batch_col_index(in, st->result) >= 0) {
+        char msg[256];
+        snprintf(msg, sizeof(msg), "source-name result column already exists: %s", st->result);
+        tf_set_last_error(msg);
+        return TF_ERROR;
+    }
+
+    const char *extra_names[1] = {st->result};
+    tf_type extra_types[1] = {TF_TYPE_STRING};
+    tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows > 0 ? in->n_rows : 1);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    const char *value = st->default_value ? st->default_value : "";
+    if (side && side->source_name && side->source_name[0]) value = side->source_name;
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_set_string(ob, r, in->n_cols, value) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+
+    if (ob->n_rows > 0) {
+        *out = ob;
+    } else {
+        tf_batch_free(ob);
+    }
+    return TF_OK;
+}
+
+static int source_name_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self;
+    (void)side;
+    *out = NULL;
+    return TF_OK;
+}
+
+static void source_name_destroy(tf_step *self) {
+    if (!self) return;
+    source_name_state *st = self->state;
+    if (st) {
+        free(st->result);
+        free(st->default_value);
+        free(st);
+    }
+    free(self);
+}
+
+tf_step *tf_source_name_create(const cJSON *args) {
+    const char *result = "_source";
+    const char *default_value = "";
+    if (args) {
+        cJSON *res = cJSON_GetObjectItemCaseSensitive(args, "result");
+        if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(args, "column");
+        if (!cJSON_IsString(res)) res = cJSON_GetObjectItemCaseSensitive(args, "as");
+        if (cJSON_IsString(res) && res->valuestring && res->valuestring[0]) result = res->valuestring;
+        cJSON *def = cJSON_GetObjectItemCaseSensitive(args, "default");
+        if (cJSON_IsString(def) && def->valuestring) default_value = def->valuestring;
+    }
+
+    source_name_state *st = calloc(1, sizeof(source_name_state));
+    if (!st) return NULL;
+    st->result = strdup(result);
+    st->default_value = strdup(default_value);
+    if (!st->result || !st->default_value) {
+        free(st->result);
+        free(st->default_value);
+        free(st);
+        return NULL;
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) {
+        free(st->result);
+        free(st->default_value);
+        free(st);
+        return NULL;
+    }
+    step->process = source_name_process;
+    step->flush = source_name_flush;
+    step->destroy = source_name_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_split.c

@@ -0,0 +1,151 @@
+/*
+ * op_split.c — Split column into multiple columns.
+ *
+ * Config: {"column": "name", "delimiter": " ", "names": ["first", "last"]}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+typedef struct {
+    char  *column;
+    char  *delimiter;
+    char **names;
+    size_t n_names;
+} split_state;
+
+static int split_process(tf_step *self, tf_batch *in, tf_batch **out,
+                         tf_side_channels *side) {
+    (void)side;
+    split_state *st = self->state;
+    *out = NULL;
+
+    size_t out_cols = 0;
+    if (tf_size_add(in->n_cols, st->n_names, &out_cols) != TF_OK) return TF_ERROR;
+    tf_batch *ob = tf_batch_create(out_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+
+    /* Copy existing schema */
+    for (size_t c = 0; c < in->n_cols; c++) {
+        if (tf_batch_set_schema(ob, c, in->col_names[c], in->col_types[c]) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+    /* Add new columns */
+    for (size_t k = 0; k < st->n_names; k++) {
+        if (tf_batch_set_schema(ob, in->n_cols + k, st->names[k], TF_TYPE_STRING) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+
+    int ci = tf_batch_col_index(in, st->column);
+    size_t delim_len = strlen(st->delimiter);
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        /* Initialize new columns as null */
+        for (size_t k = 0; k < st->n_names; k++) {
+            if (tf_batch_set_null(ob, r, in->n_cols + k) != TF_OK) {
+                tf_batch_free(ob);
+                return TF_ERROR;
+            }
+        }
+
+        if (ci >= 0 && !tf_batch_is_null(in, r, ci) && in->col_types[ci] == TF_TYPE_STRING) {
+            const char *val = tf_batch_get_string(in, r, ci);
+            const char *p = val ? val : "";
+            for (size_t k = 0; k < st->n_names && *p; k++) {
+                const char *found = strstr(p, st->delimiter);
+                size_t tok_len = found ? (size_t)(found - p) : strlen(p);
+                char *tok = malloc(tok_len + 1);
+                if (!tok) {
+                    tf_batch_free(ob);
+                    return TF_ERROR;
+                }
+                memcpy(tok, p, tok_len);
+                tok[tok_len] = '\0';
+                int rc = tf_batch_set_string(ob, r, in->n_cols + k, tok);
+                free(tok);
+                if (rc != TF_OK) {
+                    tf_batch_free(ob);
+                    return TF_ERROR;
+                }
+                if (found) p = found + delim_len;
+                else break;
+            }
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int split_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void split_state_free(split_state *st) {
+    if (!st) return;
+    free(st->column);
+    free(st->delimiter);
+    if (st->names) {
+        for (size_t i = 0; i < st->n_names; i++) free(st->names[i]);
+    }
+    free(st->names);
+    free(st);
+}
+
+static void split_destroy(tf_step *self) {
+    split_state_free(self ? self->state : NULL);
+    free(self);
+}
+
+tf_step *tf_split_create(const cJSON *args) {
+    if (!args) return NULL;
+    cJSON *col_j = cJSON_GetObjectItemCaseSensitive(args, "column");
+    cJSON *names_j = cJSON_GetObjectItemCaseSensitive(args, "names");
+    if (!cJSON_IsString(col_j) || !cJSON_IsArray(names_j)) return NULL;
+
+    int n = cJSON_GetArraySize(names_j);
+    if (n <= 0) return NULL;
+
+    split_state *st = calloc(1, sizeof(split_state));
+    if (!st) return NULL;
+    st->column = strdup(col_j->valuestring);
+    if (!st->column) { split_state_free(st); return NULL; }
+
+    cJSON *delim_j = cJSON_GetObjectItemCaseSensitive(args, "delimiter");
+    const char *delim = cJSON_IsString(delim_j) ? delim_j->valuestring : " ";
+    if (delim[0] == '\0') { split_state_free(st); return NULL; }
+    st->delimiter = strdup(delim);
+    if (!st->delimiter) { split_state_free(st); return NULL; }
+
+    st->names = calloc((size_t)n, sizeof(char *));
+    if (!st->names) { split_state_free(st); return NULL; }
+    st->n_names = (size_t)n;
+    for (int i = 0; i < n; i++) {
+        cJSON *item = cJSON_GetArrayItem(names_j, i);
+        if (!cJSON_IsString(item)) { split_state_free(st); return NULL; }
+        st->names[i] = strdup(item->valuestring);
+        if (!st->names[i]) { split_state_free(st); return NULL; }
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { split_state_free(st); return NULL; }
+    step->process = split_process;
+    step->flush = split_flush;
+    step->destroy = split_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_split_data.c

@@ -0,0 +1,119 @@
+/*
+ * op_split_data.c — Train/test split with probabilistic assignment.
+ * Uses LCG PRNG for reproducible splitting.
+ *
+ * Config: {"ratio": 0.8, "result": "_split", "seed": 42}
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char   *result;
+    double  ratio;
+    uint64_t seed;
+    uint64_t row_index;
+} split_data_state;
+
+/* Simple LCG for deterministic random */
+static double lcg_random(uint64_t seed, uint64_t index) {
+    uint64_t x = seed ^ (index * 6364136223846793005ULL + 1442695040888963407ULL);
+    x = x * 6364136223846793005ULL + 1442695040888963407ULL;
+    x = x * 6364136223846793005ULL + 1442695040888963407ULL;
+    return (double)(x >> 33) / (double)(1ULL << 31);
+}
+
+static int split_data_process(tf_step *self, tf_batch *in, tf_batch **out,
+                              tf_side_channels *side) {
+    (void)side;
+    split_data_state *st = self->state;
+    *out = NULL;
+
+    const char *extra_names[1] = {st->result};
+    tf_type extra_types[1] = {TF_TYPE_STRING};
+    tf_batch *ob = tf_batch_create(in->n_cols + 1, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_with_extra_cols(ob, in, extra_names, extra_types, 1) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+
+        double rval = lcg_random(st->seed, st->row_index);
+        const char *label = (rval < st->ratio) ? "train" : "test";
+        if (tf_batch_set_string(ob, r, in->n_cols, label) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        st->row_index++;
+    }
+
+    *out = ob;
+    return TF_OK;
+}
+
+static int split_data_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    (void)self; (void)side; *out = NULL; return TF_OK;
+}
+
+static void split_data_destroy(tf_step *self) {
+    split_data_state *st = self->state;
+    if (st) { free(st->result); free(st); }
+    free(self);
+}
+
+tf_step *tf_split_data_create(const cJSON *args) {
+    if (!args) return NULL;
+
+    split_data_state *st = calloc(1, sizeof(split_data_state));
+    if (!st) return NULL;
+
+    cJSON *ratio_j = cJSON_GetObjectItemCaseSensitive(args, "ratio");
+    st->ratio = 0.8;
+    if (ratio_j) {
+        if (!cJSON_IsNumber(ratio_j) || !isfinite(ratio_j->valuedouble) ||
+            ratio_j->valuedouble < 0.0 || ratio_j->valuedouble > 1.0) {
+            tf_set_last_error("split-data: ratio must be a finite number between 0 and 1");
+            free(st);
+            return NULL;
+        }
+        st->ratio = ratio_j->valuedouble;
+    }
+
+    cJSON *seed_j = cJSON_GetObjectItemCaseSensitive(args, "seed");
+    st->seed = 42;
+    if (seed_j) {
+        size_t parsed_seed = 0;
+        if (tf_json_size_value(seed_j, "seed", 0, TF_MAX_SAFE_SIZE_ARG,
+                               &parsed_seed, "split-data") < 0) {
+            free(st);
+            return NULL;
+        }
+        st->seed = (uint64_t)parsed_seed;
+    }
+
+    cJSON *res_j = cJSON_GetObjectItemCaseSensitive(args, "result");
+    st->result = strdup(cJSON_IsString(res_j) ? res_j->valuestring : "_split");
+    if (!st->result) { free(st); return NULL; }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { free(st->result); free(st); return NULL; }
+    step->process = split_data_process;
+    step->flush = split_data_flush;
+    step->destroy = split_data_destroy;
+    step->state = st;
+    return step;
+}

package/csrc/op_stack.c

@@ -0,0 +1,271 @@
+/*
+ * op_stack.c — Vertically concatenate a second CSV file into the stream.
+ *
+ * Passes through all input batches, then on flush reads and appends
+ * rows from a second CSV file. Optionally adds a tag column to
+ * identify the source.
+ *
+ * Args:
+ *   file (string, required) — path to CSV file to append
+ *   tag (string, optional) — name of source-identifying column
+ *   tag_value (string, optional) — value for tag column on appended rows
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+typedef struct {
+    char *file_path;
+    char *validated_file_path;
+    char *tag_col;       /* NULL if no tag */
+    char *tag_value;     /* value for appended rows */
+    char *tag_value_in;  /* value for passthrough rows (filename or "input") */
+    FILE *file;
+    tf_decoder *decoder;
+    tf_batch **pending_batches;
+    size_t n_pending_batches;
+    size_t pending_batch_index;
+    int file_started;
+    int file_done;
+} stack_state;
+
+static void stack_clear_pending(stack_state *st) {
+    if (!st || !st->pending_batches) return;
+    tf_batch_array_free_items(st->pending_batches + st->pending_batch_index,
+                              st->n_pending_batches - st->pending_batch_index);
+    free(st->pending_batches);
+    st->pending_batches = NULL;
+    st->n_pending_batches = 0;
+    st->pending_batch_index = 0;
+}
+
+static void stack_close_file_reader(stack_state *st) {
+    if (!st) return;
+    stack_clear_pending(st);
+    if (st->decoder) {
+        st->decoder->destroy(st->decoder);
+        st->decoder = NULL;
+    }
+    if (st->file) {
+        fclose(st->file);
+        st->file = NULL;
+    }
+}
+
+static void stack_state_free(stack_state *st) {
+    if (!st) return;
+    stack_close_file_reader(st);
+    free(st->file_path);
+    free(st->validated_file_path);
+    free(st->tag_col);
+    free(st->tag_value);
+    free(st->tag_value_in);
+    free(st);
+}
+
+static void stack_destroy(tf_step *self) {
+    if (self) stack_state_free(self->state);
+    free(self);
+}
+
+TF_WARN_UNUSED static int stack_write_error(tf_side_channels *side, const char *msg) {
+    return tf_side_write_error(side, msg);
+}
+
+static int stack_open_file_reader(stack_state *st, tf_side_channels *side) {
+    if (st->file_started) return TF_OK;
+    st->file_started = 1;
+    st->file = tf_policy_fopen_read(st->file_path, st->validated_file_path);
+    if (!st->file) {
+        if (stack_write_error(side, "stack: cannot open file") != TF_OK) return TF_ERROR;
+        return TF_ERROR;
+    }
+    st->decoder = tf_csv_decoder_create(NULL);
+    if (!st->decoder) {
+        stack_close_file_reader(st);
+        return TF_ERROR;
+    }
+    return TF_OK;
+}
+
+static int stack_take_pending_batch(stack_state *st, tf_batch **out) {
+    if (!st->pending_batches || st->pending_batch_index >= st->n_pending_batches) {
+        stack_clear_pending(st);
+        return 0;
+    }
+    *out = st->pending_batches[st->pending_batch_index];
+    st->pending_batches[st->pending_batch_index] = NULL;
+    st->pending_batch_index++;
+    if (st->pending_batch_index >= st->n_pending_batches) stack_clear_pending(st);
+    return 1;
+}
+
+static int stack_next_decoded_file_batch(stack_state *st, tf_batch **out,
+                                         tf_side_channels *side) {
+    *out = NULL;
+    if (stack_take_pending_batch(st, out)) return TF_OK;
+    if (st->file_done) return TF_OK;
+    if (stack_open_file_reader(st, side) != TF_OK) return TF_ERROR;
+
+    for (;;) {
+        uint8_t buf[64 * 1024];
+        size_t n = fread(buf, 1, sizeof(buf), st->file);
+        tf_batch **batches = NULL;
+        size_t n_batches = 0;
+        int rc = TF_OK;
+
+        if (n > 0) {
+            rc = st->decoder->decode(st->decoder, buf, n, &batches, &n_batches, side);
+        } else {
+            if (ferror(st->file)) {
+                int err_rc = stack_write_error(side, "stack: failed reading file");
+                stack_close_file_reader(st);
+                st->file_done = 1;
+                if (err_rc != TF_OK) return TF_ERROR;
+                return TF_ERROR;
+            }
+            rc = st->decoder->flush(st->decoder, &batches, &n_batches, side);
+            st->file_done = 1;
+            stack_close_file_reader(st);
+        }
+        if (rc != TF_OK) {
+            tf_batch_array_free(batches, n_batches);
+            stack_close_file_reader(st);
+            st->file_done = 1;
+            return TF_ERROR;
+        }
+        st->pending_batches = batches;
+        st->n_pending_batches = n_batches;
+        st->pending_batch_index = 0;
+        if (stack_take_pending_batch(st, out)) return TF_OK;
+        if (st->file_done) return TF_OK;
+    }
+}
+
+/*
+ * Add tag column to a batch. Returns a new batch with the tag column prepended.
+ */
+static tf_batch *add_tag_column(tf_batch *in, const char *tag_col, const char *tag_value) {
+    size_t out_cols = 0;
+    if (tf_size_add(in->n_cols, 1, &out_cols) != TF_OK) return NULL;
+    tf_batch *out = tf_batch_create(out_cols, in->n_rows);
+    if (!out) return NULL;
+
+    if (tf_batch_set_schema(out, 0, tag_col, TF_TYPE_STRING) != TF_OK) goto fail;
+    for (size_t c = 0; c < in->n_cols; c++) {
+        if (tf_batch_set_schema(out, c + 1, in->col_names[c], in->col_types[c]) != TF_OK) goto fail;
+    }
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_ensure_capacity(out, r + 1) != TF_OK) goto fail;
+        if (tf_batch_set_string(out, r, 0, tag_value ? tag_value : "") != TF_OK) goto fail;
+        for (size_t c = 0; c < in->n_cols; c++) {
+            if (tf_batch_copy_cell(out, r, c + 1, in, r, c) != TF_OK) goto fail;
+        }
+        if (tf_batch_expose_row(out, r) != TF_OK) goto fail;
+    }
+    return out;
+
+fail:
+    tf_batch_free(out);
+    return NULL;
+}
+
+static int stack_process(tf_step *self, tf_batch *in, tf_batch **out,
+                         tf_side_channels *side) {
+    stack_state *st = self->state;
+    (void)side;
+    *out = NULL;
+
+    if (st->tag_col) {
+        *out = add_tag_column(in, st->tag_col, st->tag_value_in);
+        return *out ? TF_OK : TF_ERROR;
+    }
+
+    tf_batch *ob = tf_batch_create(in->n_cols, in->n_rows);
+    if (!ob) return TF_ERROR;
+    if (tf_batch_clone_schema(ob, in) != TF_OK) {
+        tf_batch_free(ob);
+        return TF_ERROR;
+    }
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (tf_batch_copy_row(ob, r, in, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+        if (tf_batch_expose_row(ob, r) != TF_OK) {
+            tf_batch_free(ob);
+            return TF_ERROR;
+        }
+    }
+    *out = ob;
+    return TF_OK;
+}
+
+static int stack_flush_next(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    stack_state *st = self->state;
+    *out = NULL;
+
+    for (;;) {
+        tf_batch *batch = NULL;
+        if (stack_next_decoded_file_batch(st, &batch, side) != TF_OK) return TF_ERROR;
+        if (!batch) return TF_OK;
+        if (batch->n_rows == 0) {
+            tf_batch_free(batch);
+            continue;
+        }
+        if (!st->tag_col) {
+            *out = batch;
+            return TF_OK;
+        }
+        *out = add_tag_column(batch, st->tag_col, st->tag_value);
+        tf_batch_free(batch);
+        return *out ? TF_OK : TF_ERROR;
+    }
+}
+
+static int stack_flush(tf_step *self, tf_batch **out, tf_side_channels *side) {
+    return stack_flush_next(self, out, side);
+}
+
+tf_step *tf_stack_create(const cJSON *args) {
+    if (!args) return NULL;
+
+    cJSON *file = cJSON_GetObjectItemCaseSensitive(args, "file");
+    if (!cJSON_IsString(file) || !file->valuestring[0]) return NULL;
+
+    stack_state *st = calloc(1, sizeof(stack_state));
+    if (!st) return NULL;
+
+    st->file_path = strdup(file->valuestring);
+    if (!st->file_path) { stack_state_free(st); return NULL; }
+    const char *validated = tf_policy_validated_path_arg(args, "file");
+    if (validated) {
+        st->validated_file_path = strdup(validated);
+        if (!st->validated_file_path) { stack_state_free(st); return NULL; }
+    }
+
+    cJSON *tag = cJSON_GetObjectItemCaseSensitive(args, "tag");
+    if (cJSON_IsString(tag) && tag->valuestring[0]) {
+        st->tag_col = strdup(tag->valuestring);
+        cJSON *tv = cJSON_GetObjectItemCaseSensitive(args, "tag_value");
+        st->tag_value = strdup(cJSON_IsString(tv) ? tv->valuestring : st->file_path);
+        st->tag_value_in = strdup("input");
+        if (!st->tag_col || !st->tag_value || !st->tag_value_in) {
+            stack_state_free(st);
+            return NULL;
+        }
+    }
+
+    tf_step *step = calloc(1, sizeof(tf_step));
+    if (!step) { stack_state_free(st); return NULL; }
+    step->process = stack_process;
+    step->flush = stack_flush;
+    step->flush_next = stack_flush_next;
+    step->destroy = stack_destroy;
+    step->state = st;
+    return step;
+}