tranfi 0.0.2 → 0.1.2

package/csrc/pipeline.c

@@ -0,0 +1,315 @@
+/*
+ * pipeline.c — Pipeline orchestrator.
+ *
+ * Creates a pipeline from a JSON plan, streams bytes through
+ * decode → steps → encode, and routes output to channels.
+ */
+
+#include "internal.h"
+#include "dsl.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#define TRANFI_VERSION "0.1.0"
+
+static char *g_last_error = NULL;
+
+void tf_set_last_error(const char *msg) {
+    free(g_last_error);
+    g_last_error = msg ? strdup(msg) : NULL;
+}
+
+const char *tf_last_error(void) {
+    return g_last_error;
+}
+
+const char *tf_version(void) {
+    return TRANFI_VERSION;
+}
+
+static tf_pipeline *assemble_pipeline(tf_decoder *decoder, tf_step **steps,
+                                      size_t n_steps, tf_encoder *encoder) {
+    tf_pipeline *p = calloc(1, sizeof(tf_pipeline));
+    if (!p) {
+        decoder->destroy(decoder);
+        encoder->destroy(encoder);
+        for (size_t i = 0; i < n_steps; i++) steps[i]->destroy(steps[i]);
+        free(steps);
+        tf_set_last_error("out of memory");
+        return NULL;
+    }
+
+    p->decoder = decoder;
+    p->steps = steps;
+    p->n_steps = n_steps;
+    p->encoder = encoder;
+
+    for (int i = 0; i < TF_NUM_CHANNELS; i++) {
+        tf_buffer_init(&p->output[i]);
+    }
+
+    /* Wire up side channels */
+    p->side.errors = &p->output[TF_CHAN_ERRORS];
+    p->side.stats = &p->output[TF_CHAN_STATS];
+    p->side.samples = &p->output[TF_CHAN_SAMPLES];
+
+    return p;
+}
+
+tf_pipeline *tf_pipeline_create(const char *plan_json, size_t len) {
+    if (!plan_json || len == 0) {
+        tf_set_last_error("empty plan");
+        return NULL;
+    }
+
+    char *error = NULL;
+
+    /* 1. Parse JSON → IR */
+    tf_ir_plan *ir = tf_ir_from_json(plan_json, len, &error);
+    if (!ir) {
+        tf_set_last_error(error ? error : "failed to parse plan");
+        free(error);
+        return NULL;
+    }
+
+    /* 2. Validate */
+    if (tf_ir_validate(ir) != TF_OK) {
+        tf_set_last_error(ir->error ? ir->error : "validation failed");
+        tf_ir_plan_free(ir);
+        return NULL;
+    }
+
+    /* 3. Schema inference (best-effort, non-fatal) */
+    tf_ir_infer_schema(ir);
+
+    /* 4. Compile to native target */
+    tf_decoder *decoder = NULL;
+    tf_step **steps = NULL;
+    size_t n_steps = 0;
+    tf_encoder *encoder = NULL;
+    if (tf_compile_native(ir, &decoder, &steps, &n_steps, &encoder, &error) != TF_OK) {
+        tf_set_last_error(error ? error : "compilation failed");
+        free(error);
+        tf_ir_plan_free(ir);
+        return NULL;
+    }
+
+    tf_ir_plan_free(ir);
+
+    /* 5. Assemble pipeline */
+    return assemble_pipeline(decoder, steps, n_steps, encoder);
+}
+
+tf_pipeline *tf_pipeline_create_from_ir(const tf_ir_plan *plan) {
+    if (!plan) {
+        tf_set_last_error("NULL IR plan");
+        return NULL;
+    }
+
+    char *error = NULL;
+    tf_decoder *decoder = NULL;
+    tf_step **steps = NULL;
+    size_t n_steps = 0;
+    tf_encoder *encoder = NULL;
+
+    if (tf_compile_native(plan, &decoder, &steps, &n_steps, &encoder, &error) != TF_OK) {
+        tf_set_last_error(error ? error : "compilation failed");
+        free(error);
+        return NULL;
+    }
+
+    return assemble_pipeline(decoder, steps, n_steps, encoder);
+}
+
+/* Public IR wrappers (thin forwarding to ir.h functions) */
+tf_ir_plan *tf_ir_plan_from_json(const char *json, size_t len, char **error) {
+    return tf_ir_from_json(json, len, error);
+}
+char *tf_ir_plan_to_json(const tf_ir_plan *plan) {
+    return tf_ir_to_json(plan);
+}
+int tf_ir_plan_validate(tf_ir_plan *plan) {
+    return tf_ir_validate(plan);
+}
+int tf_ir_plan_infer_schema(tf_ir_plan *plan) {
+    return tf_ir_infer_schema(plan);
+}
+void tf_ir_plan_destroy(tf_ir_plan *plan) {
+    tf_ir_plan_free(plan);
+}
+
+/*
+ * Process a batch through all steps, then encode.
+ */
+static int process_batch(tf_pipeline *p, tf_batch *batch) {
+    tf_batch *current = batch;
+    int batch_owned = 0; /* 0 = still owned by caller (decoder) */
+
+    p->rows_in += current->n_rows;
+
+    for (size_t i = 0; i < p->n_steps; i++) {
+        tf_batch *next = NULL;
+        int rc = p->steps[i]->process(p->steps[i], current, &next, &p->side);
+
+        if (batch_owned) tf_batch_free(current);
+        batch_owned = 1;
+
+        if (rc != TF_OK) return TF_ERROR;
+        if (!next) return TF_OK; /* filtered away entirely */
+        current = next;
+    }
+
+    /* Encode */
+    if (current->n_rows > 0) {
+        p->rows_out += current->n_rows;
+        int rc = p->encoder->encode(p->encoder, current, &p->output[TF_CHAN_MAIN]);
+        if (batch_owned) tf_batch_free(current);
+        return rc;
+    }
+
+    if (batch_owned) tf_batch_free(current);
+    return TF_OK;
+}
+
+int tf_pipeline_push(tf_pipeline *p, const uint8_t *data, size_t len) {
+    if (!p || p->finished) return TF_ERROR;
+
+    p->bytes_in += len;
+
+    /* Decode bytes into batches */
+    tf_batch **batches = NULL;
+    size_t n_batches = 0;
+    int rc = p->decoder->decode(p->decoder, data, len, &batches, &n_batches);
+    if (rc != TF_OK) {
+        free(p->error);
+        p->error = strdup("decode error");
+        return TF_ERROR;
+    }
+
+    /* Process each batch */
+    for (size_t i = 0; i < n_batches; i++) {
+        rc = process_batch(p, batches[i]);
+        tf_batch_free(batches[i]);
+        if (rc != TF_OK) {
+            for (size_t j = i + 1; j < n_batches; j++) tf_batch_free(batches[j]);
+            free(batches);
+            free(p->error);
+            p->error = strdup("processing error");
+            return TF_ERROR;
+        }
+    }
+    free(batches);
+
+    return TF_OK;
+}
+
+int tf_pipeline_finish(tf_pipeline *p) {
+    if (!p || p->finished) return TF_ERROR;
+    p->finished = 1;
+
+    /* Flush decoder */
+    tf_batch **batches = NULL;
+    size_t n_batches = 0;
+    int rc = p->decoder->flush(p->decoder, &batches, &n_batches);
+    if (rc == TF_OK) {
+        for (size_t i = 0; i < n_batches; i++) {
+            process_batch(p, batches[i]);
+            tf_batch_free(batches[i]);
+        }
+        free(batches);
+    }
+
+    /* Flush steps */
+    for (size_t i = 0; i < p->n_steps; i++) {
+        tf_batch *flushed = NULL;
+        rc = p->steps[i]->flush(p->steps[i], &flushed, &p->side);
+        if (rc == TF_OK && flushed) {
+            /* Run flushed batch through remaining steps */
+            tf_batch *current = flushed;
+            int owned = 1;
+            for (size_t j = i + 1; j < p->n_steps; j++) {
+                tf_batch *next = NULL;
+                p->steps[j]->process(p->steps[j], current, &next, &p->side);
+                if (owned) tf_batch_free(current);
+                owned = 1;
+                if (!next) { current = NULL; break; }
+                current = next;
+            }
+            if (current && current->n_rows > 0) {
+                p->rows_out += current->n_rows;
+                p->encoder->encode(p->encoder, current, &p->output[TF_CHAN_MAIN]);
+            }
+            if (current && owned) tf_batch_free(current);
+        }
+    }
+
+    /* Flush encoder */
+    p->encoder->flush(p->encoder, &p->output[TF_CHAN_MAIN]);
+
+    /* Emit final stats */
+    p->bytes_out = tf_buffer_readable(&p->output[TF_CHAN_MAIN]);
+    char stats_buf[256];
+    snprintf(stats_buf, sizeof(stats_buf),
+             "{\"rows_in\":%zu,\"rows_out\":%zu,\"bytes_in\":%zu,\"bytes_out\":%zu}\n",
+             p->rows_in, p->rows_out, p->bytes_in, p->bytes_out);
+    tf_buffer_write_str(&p->output[TF_CHAN_STATS], stats_buf);
+
+    return TF_OK;
+}
+
+size_t tf_pipeline_pull(tf_pipeline *p, int channel, uint8_t *buf, size_t buf_len) {
+    if (!p || channel < 0 || channel >= TF_NUM_CHANNELS) return 0;
+    return tf_buffer_read(&p->output[channel], buf, buf_len);
+}
+
+const char *tf_pipeline_error(tf_pipeline *p) {
+    return p ? p->error : NULL;
+}
+
+void tf_pipeline_free(tf_pipeline *p) {
+    if (!p) return;
+    if (p->decoder) p->decoder->destroy(p->decoder);
+    if (p->encoder) p->encoder->destroy(p->encoder);
+    for (size_t i = 0; i < p->n_steps; i++) {
+        if (p->steps[i]) p->steps[i]->destroy(p->steps[i]);
+    }
+    free(p->steps);
+    for (int i = 0; i < TF_NUM_CHANNELS; i++) {
+        tf_buffer_free(&p->output[i]);
+    }
+    free(p->error);
+    free(p);
+}
+
+char *tf_compile_to_sql(const char *dsl, size_t len, char **error) {
+    if (error) *error = NULL;
+    tf_ir_plan *plan = tf_dsl_parse(dsl, len, error);
+    if (!plan) return NULL;
+    if (tf_ir_validate(plan) != TF_OK) {
+        if (error) { free(*error); *error = strdup(plan->error ? plan->error : "validation failed"); }
+        tf_ir_plan_destroy(plan);
+        return NULL;
+    }
+    tf_ir_infer_schema(plan);
+    char *sql = tf_ir_to_sql(plan, error);
+    tf_ir_plan_destroy(plan);
+    return sql;
+}
+
+char *tf_ir_plan_to_sql(const tf_ir_plan *plan, char **error) {
+    return tf_ir_to_sql(plan, error);
+}
+
+char *tf_compile_dsl(const char *dsl, size_t len, char **error) {
+    if (error) *error = NULL;
+    tf_ir_plan *plan = tf_dsl_parse(dsl, len, error);
+    if (!plan) return NULL;
+    char *json = tf_ir_plan_to_json(plan);
+    tf_ir_plan_destroy(plan);
+    return json;
+}
+
+void tf_string_free(char *s) {
+    free(s);
+}

package/csrc/plan.c

@@ -0,0 +1,206 @@
+/*
+ * plan.c — Parse a JSON pipeline plan and instantiate decoder, steps, encoder.
+ *
+ * Plan format:
+ * {
+ *   "steps": [
+ *     {"op": "codec.csv.decode", "args": {"delimiter": ","}},
+ *     {"op": "filter", "args": {"expr": "col('age') > 25"}},
+ *     {"op": "codec.csv.encode", "args": {}}
+ *   ]
+ * }
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static void set_error(char **error, const char *msg) {
+    if (error) {
+        free(*error);
+        size_t len = strlen(msg) + 1;
+        *error = malloc(len);
+        if (*error) memcpy(*error, msg, len);
+    }
+}
+
+static void set_errorf(char **error, const char *fmt, const char *detail) {
+    if (error) {
+        char buf[256];
+        snprintf(buf, sizeof(buf), fmt, detail);
+        set_error(error, buf);
+    }
+}
+
+int tf_plan_parse(const char *json, size_t len,
+                  tf_decoder **out_decoder, tf_step ***out_steps, size_t *out_n_steps,
+                  tf_encoder **out_encoder, char **error) {
+    *out_decoder = NULL;
+    *out_steps = NULL;
+    *out_n_steps = 0;
+    *out_encoder = NULL;
+
+    /* Parse JSON */
+    cJSON *root = cJSON_ParseWithLength(json, len);
+    if (!root) {
+        set_error(error, "invalid JSON in plan");
+        return TF_ERROR;
+    }
+
+    cJSON *steps_arr = cJSON_GetObjectItemCaseSensitive(root, "steps");
+    if (!cJSON_IsArray(steps_arr)) {
+        set_error(error, "plan must have a 'steps' array");
+        cJSON_Delete(root);
+        return TF_ERROR;
+    }
+
+    int n_json_steps = cJSON_GetArraySize(steps_arr);
+    if (n_json_steps == 0) {
+        set_error(error, "plan has no steps");
+        cJSON_Delete(root);
+        return TF_ERROR;
+    }
+
+    /* Temporary arrays for transforms (max = n_json_steps) */
+    tf_step **steps = calloc(n_json_steps, sizeof(tf_step *));
+    size_t n_steps = 0;
+    tf_decoder *decoder = NULL;
+    tf_encoder *encoder = NULL;
+    int err = TF_OK;
+
+    for (int i = 0; i < n_json_steps; i++) {
+        cJSON *step_json = cJSON_GetArrayItem(steps_arr, i);
+        cJSON *op_json = cJSON_GetObjectItemCaseSensitive(step_json, "op");
+        cJSON *args_json = cJSON_GetObjectItemCaseSensitive(step_json, "args");
+
+        if (!cJSON_IsString(op_json)) {
+            set_errorf(error, "step %d missing 'op' string", "");
+            err = TF_ERROR;
+            break;
+        }
+        const char *op = op_json->valuestring;
+
+        /* Match op to constructor */
+        if (strcmp(op, "codec.csv.decode") == 0) {
+            if (decoder) {
+                set_error(error, "multiple decoders not supported");
+                err = TF_ERROR;
+                break;
+            }
+            decoder = tf_csv_decoder_create(args_json);
+            if (!decoder) {
+                set_error(error, "failed to create CSV decoder");
+                err = TF_ERROR;
+                break;
+            }
+        } else if (strcmp(op, "codec.csv.encode") == 0) {
+            if (encoder) {
+                set_error(error, "multiple encoders not supported");
+                err = TF_ERROR;
+                break;
+            }
+            encoder = tf_csv_encoder_create(args_json);
+            if (!encoder) {
+                set_error(error, "failed to create CSV encoder");
+                err = TF_ERROR;
+                break;
+            }
+        } else if (strcmp(op, "codec.jsonl.decode") == 0) {
+            if (decoder) {
+                set_error(error, "multiple decoders not supported");
+                err = TF_ERROR;
+                break;
+            }
+            decoder = tf_jsonl_decoder_create(args_json);
+            if (!decoder) {
+                set_error(error, "failed to create JSONL decoder");
+                err = TF_ERROR;
+                break;
+            }
+        } else if (strcmp(op, "codec.jsonl.encode") == 0) {
+            if (encoder) {
+                set_error(error, "multiple encoders not supported");
+                err = TF_ERROR;
+                break;
+            }
+            encoder = tf_jsonl_encoder_create(args_json);
+            if (!encoder) {
+                set_error(error, "failed to create JSONL encoder");
+                err = TF_ERROR;
+                break;
+            }
+        } else if (strcmp(op, "filter") == 0) {
+            tf_step *s = tf_filter_create(args_json);
+            if (!s) {
+                set_error(error, "failed to create filter step");
+                err = TF_ERROR;
+                break;
+            }
+            steps[n_steps++] = s;
+        } else if (strcmp(op, "select") == 0) {
+            tf_step *s = tf_select_create(args_json);
+            if (!s) {
+                set_error(error, "failed to create select step");
+                err = TF_ERROR;
+                break;
+            }
+            steps[n_steps++] = s;
+        } else if (strcmp(op, "rename") == 0) {
+            tf_step *s = tf_rename_create(args_json);
+            if (!s) {
+                set_error(error, "failed to create rename step");
+                err = TF_ERROR;
+                break;
+            }
+            steps[n_steps++] = s;
+        } else if (strcmp(op, "head") == 0) {
+            tf_step *s = tf_head_create(args_json);
+            if (!s) {
+                set_error(error, "failed to create head step");
+                err = TF_ERROR;
+                break;
+            }
+            steps[n_steps++] = s;
+        } else {
+            set_errorf(error, "unknown op: '%s'", op);
+            err = TF_ERROR;
+            break;
+        }
+    }
+
+    cJSON_Delete(root);
+
+    if (err != TF_OK) {
+        /* Cleanup on error */
+        if (decoder) decoder->destroy(decoder);
+        if (encoder) encoder->destroy(encoder);
+        for (size_t i = 0; i < n_steps; i++) {
+            if (steps[i]) steps[i]->destroy(steps[i]);
+        }
+        free(steps);
+        return TF_ERROR;
+    }
+
+    if (!decoder) {
+        set_error(error, "plan has no decoder (need a codec.*.decode step)");
+        if (encoder) encoder->destroy(encoder);
+        for (size_t i = 0; i < n_steps; i++) steps[i]->destroy(steps[i]);
+        free(steps);
+        return TF_ERROR;
+    }
+    if (!encoder) {
+        set_error(error, "plan has no encoder (need a codec.*.encode step)");
+        decoder->destroy(decoder);
+        for (size_t i = 0; i < n_steps; i++) steps[i]->destroy(steps[i]);
+        free(steps);
+        return TF_ERROR;
+    }
+
+    *out_decoder = decoder;
+    *out_steps = steps;
+    *out_n_steps = n_steps;
+    *out_encoder = encoder;
+    return TF_OK;
+}

package/csrc/recipes.c

@@ -0,0 +1,102 @@
+/*
+ * recipes.c — Built-in named recipes (20 common ETL pipelines).
+ */
+
+#include "recipes.h"
+#include <string.h>
+#include <ctype.h>
+
+typedef struct {
+    const char *name;
+    const char *dsl;
+    const char *description;
+} recipe_entry;
+
+static const recipe_entry recipes[] = {
+    /* ---- Data Exploration ---- */
+    {"profile",     "csv | stats | csv",
+     "Full data profiling (all statistics per column)"},
+    {"preview",     "csv | head 10 | csv",
+     "Quick preview of first 10 rows"},
+    {"schema",      "csv | head 0 | csv",
+     "Show column names only"},
+    {"summary",     "csv | stats count,min,max,avg,stddev | csv",
+     "Summary statistics"},
+    {"count",       "csv | stats count | csv",
+     "Row count per column"},
+    {"cardinality", "csv | stats count,distinct | csv",
+     "Unique value counts per column"},
+    {"distro",      "csv | stats min,p25,median,p75,max | csv",
+     "Five-number summary (quartiles)"},
+
+    /* ---- Data Quality ---- */
+    {"freq",        "csv | frequency | csv",
+     "Value frequency distribution"},
+    {"dedup",       "csv | dedup | csv",
+     "Remove duplicate rows"},
+    {"clean",       "csv | trim | csv",
+     "Trim whitespace from all columns"},
+
+    /* ---- Data Sampling ---- */
+    {"sample",      "csv | sample 100 | csv",
+     "Random sample of 100 rows"},
+    {"head",        "csv | head 20 | csv",
+     "First 20 rows"},
+    {"tail",        "csv | tail 20 | csv",
+     "Last 20 rows"},
+
+    /* ---- Format Conversion ---- */
+    {"csv2json",    "csv | jsonl",
+     "Convert CSV to JSONL"},
+    {"json2csv",    "jsonl | csv",
+     "Convert JSONL to CSV"},
+    {"tsv2csv",     "csv delimiter=\"\t\" | csv",
+     "Convert TSV to CSV"},
+    {"csv2tsv",     "csv | csv delimiter=\"\t\"",
+     "Convert CSV to TSV"},
+
+    /* ---- Display ---- */
+    {"look",        "csv | table",
+     "Pretty-print as Markdown table"},
+
+    /* ---- Analysis ---- */
+    {"histogram",   "csv | stats hist | csv",
+     "Distribution histograms"},
+    {"hash",        "csv | hash | csv",
+     "Add row hash column for change detection"},
+    {"samples",     "csv | stats sample | csv",
+     "Show sample values per column"},
+};
+
+#define RECIPE_COUNT (sizeof(recipes) / sizeof(recipes[0]))
+
+size_t tf_recipe_count(void) {
+    return RECIPE_COUNT;
+}
+
+const char *tf_recipe_name(size_t index) {
+    return index < RECIPE_COUNT ? recipes[index].name : NULL;
+}
+
+const char *tf_recipe_dsl(size_t index) {
+    return index < RECIPE_COUNT ? recipes[index].dsl : NULL;
+}
+
+const char *tf_recipe_description(size_t index) {
+    return index < RECIPE_COUNT ? recipes[index].description : NULL;
+}
+
+const char *tf_recipe_find_dsl(const char *name) {
+    if (!name) return NULL;
+    for (size_t i = 0; i < RECIPE_COUNT; i++) {
+        /* Case-insensitive comparison */
+        const char *a = name;
+        const char *b = recipes[i].name;
+        while (*a && *b && tolower((unsigned char)*a) == tolower((unsigned char)*b)) {
+            a++;
+            b++;
+        }
+        if (*a == '\0' && *b == '\0') return recipes[i].dsl;
+    }
+    return NULL;
+}

package/csrc/recipes.h

@@ -0,0 +1,27 @@
+/*
+ * recipes.h — Built-in named recipes.
+ *
+ * 20 pre-built DSL pipelines for common ETL operations:
+ *   tranfi profile    → full data profiling
+ *   tranfi preview    → first 10 rows
+ *   tranfi csv2json   → format conversion
+ *   ...
+ */
+
+#ifndef TF_RECIPES_H
+#define TF_RECIPES_H
+
+#include <stddef.h>
+
+/* Number of built-in recipes. */
+size_t tf_recipe_count(void);
+
+/* Accessors by index (0-based). Return NULL if index out of range. */
+const char *tf_recipe_name(size_t index);
+const char *tf_recipe_dsl(size_t index);
+const char *tf_recipe_description(size_t index);
+
+/* Lookup by name (case-insensitive). Returns DSL string or NULL. */
+const char *tf_recipe_find_dsl(const char *name);
+
+#endif /* TF_RECIPES_H */