tranfi 0.0.2 → 0.2.0

package/csrc/ir.c

@@ -0,0 +1,166 @@
+/*
+ * ir.c — IR plan construction, cloning, and memory management.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+/* ---- Schema helpers ---- */
+
+void tf_schema_free(tf_schema *s) {
+    if (!s) return;
+    if (s->col_names) {
+        for (size_t i = 0; i < s->n_cols; i++) {
+            free(s->col_names[i]);
+        }
+    }
+    free(s->col_names);
+    free(s->col_types);
+    s->col_names = NULL;
+    s->col_types = NULL;
+    s->n_cols = 0;
+    s->known = false;
+}
+
+int tf_schema_copy_checked(tf_schema *dst, const tf_schema *src) {
+    if (!dst) return TF_ERROR;
+    dst->known = false;
+    dst->n_cols = 0;
+    dst->col_names = NULL;
+    dst->col_types = NULL;
+    if (!src) return TF_OK;
+
+    dst->known = src->known;
+    dst->n_cols = src->n_cols;
+    if (src->n_cols == 0 || !src->known) return TF_OK;
+
+    dst->col_names = calloc(src->n_cols, sizeof(char *));
+    dst->col_types = calloc(src->n_cols, sizeof(tf_type));
+    if (!dst->col_names || !dst->col_types) {
+        tf_schema_free(dst);
+        return TF_ERROR;
+    }
+    for (size_t i = 0; i < src->n_cols; i++) {
+        dst->col_names[i] = strdup(src->col_names[i] ? src->col_names[i] : "");
+        if (!dst->col_names[i]) {
+            tf_schema_free(dst);
+            return TF_ERROR;
+        }
+        dst->col_types[i] = src->col_types[i];
+    }
+    dst->known = true;
+    return TF_OK;
+}
+
+void tf_schema_copy(tf_schema *dst, const tf_schema *src) {
+    (void)tf_schema_copy_checked(dst, src);
+}
+
+/* ---- IR node helpers ---- */
+
+static void ir_node_clear(tf_ir_node *node) {
+    free((char *)node->op);
+    if (node->args) cJSON_Delete(node->args);
+    tf_schema_free(&node->input_schema);
+    tf_schema_free(&node->output_schema);
+}
+
+/* ---- IR plan API ---- */
+
+tf_ir_plan *tf_ir_plan_create(void) {
+    tf_ir_plan *plan = calloc(1, sizeof(tf_ir_plan));
+    if (!plan) return NULL;
+    plan->capacity = 8;
+    plan->nodes = calloc(plan->capacity, sizeof(tf_ir_node));
+    if (!plan->nodes) { free(plan); return NULL; }
+    return plan;
+}
+
+int tf_ir_plan_add_node(tf_ir_plan *plan, const char *op, cJSON *args) {
+    if (!plan || !op) return -1;
+    if (plan->n_nodes >= plan->capacity) {
+        size_t new_cap = 0;
+        if (tf_size_mul(plan->capacity, 2, &new_cap) != TF_OK) return -1;
+        tf_ir_node *new_nodes = tf_reallocarray_checked(plan->nodes, new_cap,
+                                                        sizeof(tf_ir_node));
+        if (!new_nodes) return -1;
+        memset(new_nodes + plan->capacity, 0,
+               (new_cap - plan->capacity) * sizeof(tf_ir_node));
+        plan->nodes = new_nodes;
+        plan->capacity = new_cap;
+    }
+
+    char *op_copy = strdup(op);
+    cJSON *args_copy = args ? cJSON_Duplicate(args, 1) : cJSON_CreateObject();
+    if (!op_copy || !args_copy) {
+        free(op_copy);
+        if (args_copy) cJSON_Delete(args_copy);
+        return -1;
+    }
+
+    tf_ir_node *node = &plan->nodes[plan->n_nodes];
+    memset(node, 0, sizeof(tf_ir_node));
+    node->op = op_copy;
+    node->args = args_copy;
+    node->index = plan->n_nodes;
+
+    plan->n_nodes++;
+    plan->validated = false;
+    plan->schema_inferred = false;
+    return 0;
+}
+
+tf_ir_plan *tf_ir_plan_clone(const tf_ir_plan *plan) {
+    tf_ir_plan *clone = tf_ir_plan_create();
+    if (!clone) return NULL;
+
+    for (size_t i = 0; i < plan->n_nodes; i++) {
+        const tf_ir_node *src = &plan->nodes[i];
+        if (tf_ir_plan_add_node(clone, src->op, src->args) != 0) {
+            tf_ir_plan_free(clone);
+            return NULL;
+        }
+        /* Copy schemas if they were inferred */
+        tf_ir_node *dst = &clone->nodes[i];
+        if (tf_schema_copy_checked(&dst->input_schema, &src->input_schema) != TF_OK ||
+            tf_schema_copy_checked(&dst->output_schema, &src->output_schema) != TF_OK) {
+            tf_ir_plan_free(clone);
+            return NULL;
+        }
+        dst->caps = src->caps;
+        dst->memory_class = src->memory_class;
+        dst->emit_class = src->emit_class;
+        dst->schema_class = src->schema_class;
+        dst->state_estimate = src->state_estimate;
+    }
+
+    if (tf_schema_copy_checked(&clone->final_schema, &plan->final_schema) != TF_OK) {
+        tf_ir_plan_free(clone);
+        return NULL;
+    }
+    clone->plan_caps = plan->plan_caps;
+    clone->validated = plan->validated;
+    clone->schema_inferred = plan->schema_inferred;
+    if (plan->error) {
+        clone->error = strdup(plan->error);
+        if (!clone->error) {
+            tf_ir_plan_free(clone);
+            return NULL;
+        }
+    }
+
+    return clone;
+}
+
+void tf_ir_plan_free(tf_ir_plan *plan) {
+    if (!plan) return;
+    for (size_t i = 0; i < plan->n_nodes; i++) {
+        ir_node_clear(&plan->nodes[i]);
+    }
+    free(plan->nodes);
+    tf_schema_free(&plan->final_schema);
+    free(plan->error);
+    free(plan);
+}

package/csrc/ir.h

@@ -0,0 +1,208 @@
+/*
+ * ir.h — L2 Intermediate Representation types for the Tranfi compilation pipeline.
+ *
+ * Defines the op registry, schema, IR node, and IR plan types.
+ * The IR is the contract between authoring (L3) and execution (L1).
+ */
+
+#ifndef TF_IR_H
+#define TF_IR_H
+
+#include "config.h"
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+/* Forward declarations */
+typedef struct cJSON cJSON;
+typedef struct tf_ir_node tf_ir_node;
+typedef struct tf_schema tf_schema;
+typedef struct tf_host_policy tf_host_policy;
+
+/* ---- Capability flags ---- */
+
+#define TF_CAP_STREAMING      (1 << 0)  /* processes batches without buffering all data */
+#define TF_CAP_BOUNDED_MEMORY (1 << 1)  /* memory usage independent of input size */
+#define TF_CAP_BROWSER_SAFE   (1 << 2)  /* no filesystem/network access */
+#define TF_CAP_DETERMINISTIC  (1 << 3)  /* same input always produces same output */
+#define TF_CAP_FS             (1 << 4)  /* requires filesystem access */
+#define TF_CAP_NET            (1 << 5)  /* requires network access */
+
+/* ---- Op tier ---- */
+
+typedef enum {
+    TF_TIER_CORE,       /* built into the C library */
+    TF_TIER_ECOSYSTEM,  /* provided by host-side plugins */
+} tf_op_tier;
+
+/* ---- Op kind ---- */
+
+typedef enum {
+    TF_OP_DECODER,
+    TF_OP_ENCODER,
+    TF_OP_TRANSFORM,
+} tf_op_kind;
+
+/* ---- Streaming contract metadata ---- */
+
+typedef enum {
+    TF_MEM_ROW_LOCAL,       /* O(batch_rows * columns) */
+    TF_MEM_BOUNDED_STATE,   /* O(parameter), e.g. window K or tail N */
+    TF_MEM_KEY_STATE,       /* O(distinct keys/categories) */
+    TF_MEM_BLOCKING,        /* needs full input in native mode */
+    TF_MEM_EXTERNAL,        /* uses spill/SQL/external bounded storage */
+} tf_memory_class;
+
+typedef enum {
+    TF_EMIT_PER_BATCH,      /* can emit during push() */
+    TF_EMIT_ON_FLUSH,       /* emits only at finish(), e.g. tail/top/stats */
+    TF_EMIT_SIDE_ONLY,      /* writes side channels, not main output */
+    TF_EMIT_MIXED,          /* emits both during push() and flush/side output */
+} tf_emit_class;
+
+typedef enum {
+    TF_SCHEMA_STABLE,       /* output columns known from input schema/args */
+    TF_SCHEMA_PARAMETRIC,   /* output schema depends on explicit parameters */
+    TF_SCHEMA_DATA_DEPENDENT, /* output schema depends on observed data */
+} tf_schema_class;
+
+/* ---- Argument descriptor ---- */
+
+typedef struct {
+    const char *name;        /* "delimiter", "expr", "columns", etc. */
+    const char *type;        /* "string", "int", "bool", "string[]", "map" */
+    bool        required;
+    const char *default_val; /* JSON-encoded default, or NULL */
+} tf_arg_desc;
+
+/* ---- Value types (shared with batch system) ---- */
+
+#ifndef TF_TYPE_DEFINED
+#define TF_TYPE_DEFINED
+typedef enum tf_type {
+    TF_TYPE_NULL = 0,
+    TF_TYPE_BOOL,
+    TF_TYPE_INT64,
+    TF_TYPE_FLOAT64,
+    TF_TYPE_STRING,
+    TF_TYPE_DATE,       /* int32_t: days since 1970-01-01 */
+    TF_TYPE_TIMESTAMP,  /* int64_t: microseconds since 1970-01-01T00:00:00Z */
+} tf_type;
+#endif
+
+/* ---- Schema ---- */
+
+struct tf_schema {
+    char    **col_names;
+    tf_type  *col_types;
+    size_t    n_cols;
+    bool      known;  /* false if schema can't be determined until runtime */
+};
+
+void tf_schema_free(tf_schema *s);
+void tf_schema_copy(tf_schema *dst, const tf_schema *src);
+int  tf_schema_copy_checked(tf_schema *dst, const tf_schema *src);
+
+/* ---- Op registry entry ---- */
+
+typedef struct {
+    const char    *name;        /* "codec.csv.decode", "filter", etc. */
+    tf_op_kind     kind;
+    tf_op_tier     tier;
+    uint32_t       caps;        /* TF_CAP_* bitfield */
+    tf_memory_class memory_class;
+    tf_emit_class   emit_class;
+    tf_schema_class schema_class;
+    const char    *state_estimate; /* Big-O state retained by the op */
+    tf_arg_desc   *args;
+    size_t         n_args;
+    /* Schema transform: given input schema, compute output schema.
+     * Returns TF_OK or TF_ERROR. */
+    int (*infer_schema)(const tf_ir_node *node,
+                        const tf_schema *in, tf_schema *out);
+    /* Native target constructor (NULL for ecosystem ops).
+     * Returns an opaque pointer (tf_decoder*, tf_step*, or tf_encoder*). */
+    void *(*create_native)(const cJSON *args);
+} tf_op_entry;
+
+/* ---- Op registry API ---- */
+
+const tf_op_entry *tf_op_registry_find(const char *name);
+size_t             tf_op_registry_count(void);
+const tf_op_entry *tf_op_registry_get(size_t index);
+const char        *tf_memory_class_name(tf_memory_class cls);
+const char        *tf_emit_class_name(tf_emit_class cls);
+const char        *tf_schema_class_name(tf_schema_class cls);
+
+/* ---- IR node ---- */
+
+struct tf_ir_node {
+    const char     *op;           /* op name (owned, freed with node) */
+    cJSON          *args;         /* argument values (owned, freed with node) */
+    tf_schema       input_schema;
+    tf_schema       output_schema;
+    uint32_t        caps;         /* resolved capability flags from registry */
+    tf_memory_class memory_class;
+    tf_emit_class   emit_class;
+    tf_schema_class schema_class;
+    const char     *state_estimate; /* Big-O state estimate, static string */
+    size_t          index;        /* position in plan */
+};
+
+/* ---- IR plan ---- */
+
+typedef struct tf_ir_plan {
+    tf_ir_node    *nodes;
+    size_t         n_nodes;
+    size_t         capacity;      /* allocated node slots */
+    tf_schema      final_schema;  /* schema after last transform, before encoder */
+    uint32_t       plan_caps;     /* intersection of all node caps */
+    char          *error;         /* validation error, if any */
+    bool           validated;
+    bool           schema_inferred;
+} tf_ir_plan;
+
+/* ---- IR plan API ---- */
+
+tf_ir_plan *tf_ir_plan_create(void);
+int         tf_ir_plan_add_node(tf_ir_plan *plan, const char *op, cJSON *args);
+tf_ir_plan *tf_ir_plan_clone(const tf_ir_plan *plan);
+void        tf_ir_plan_free(tf_ir_plan *plan);
+
+/* ---- IR serialization ---- */
+
+tf_ir_plan *tf_ir_from_json(const char *json, size_t len, char **error);
+char       *tf_ir_to_json(const tf_ir_plan *plan);
+char       *tf_ir_to_sql(const tf_ir_plan *plan, char **error);
+
+/* ---- IR passes ---- */
+
+int tf_ir_validate(tf_ir_plan *plan);
+int tf_ir_validate_with_host_policy(tf_ir_plan *plan, const tf_host_policy *policy);
+int tf_ir_infer_schema(tf_ir_plan *plan);
+
+/* ---- Expression eval result ---- */
+
+typedef struct tf_eval_result {
+    tf_type type;
+    union {
+        int64_t     i;     /* INT64 or TIMESTAMP (microseconds since epoch) */
+        double      f;
+        const char *s;
+        bool        b;
+        int32_t     date;  /* DATE: days since epoch */
+    };
+} tf_eval_result;
+
+/* ---- Compiler ---- */
+
+/* Forward declarations of internal types */
+typedef struct tf_decoder tf_decoder;
+typedef struct tf_step tf_step;
+typedef struct tf_encoder tf_encoder;
+
+int tf_compile_native(const tf_ir_plan *plan,
+                      tf_decoder **decoder, tf_step ***steps,
+                      size_t *n_steps, tf_encoder **encoder, char **error);
+
+#endif /* TF_IR_H */

package/csrc/ir_schema.c

@@ -0,0 +1,75 @@
+/*
+ * ir_schema.c — Forward schema inference pass.
+ *
+ * Walks nodes in execution order, calling each op's infer_schema callback
+ * to propagate schema information forward through the plan.
+ * Best-effort and non-fatal — unknown schemas propagate as known=false.
+ */
+
+#include "ir.h"
+#include <stdlib.h>
+
+#define TF_OK    0
+#define TF_ERROR (-1)
+
+int tf_ir_infer_schema(tf_ir_plan *plan) {
+    if (plan->n_nodes == 0) return TF_OK;
+
+    /* Start with an empty, unknown input schema for the first node (decoder) */
+    tf_schema current = {0};
+    current.known = false;
+
+    for (size_t i = 0; i < plan->n_nodes; i++) {
+        tf_ir_node *node = &plan->nodes[i];
+        const tf_op_entry *entry = tf_op_registry_find(node->op);
+
+        /* Copy current schema as this node's input */
+        tf_schema_free(&node->input_schema);
+        if (tf_schema_copy_checked(&node->input_schema, &current) != TF_OK) {
+            tf_schema_free(&current);
+            plan->schema_inferred = false;
+            return TF_ERROR;
+        }
+
+        if (entry && entry->infer_schema) {
+            /* Clear old output schema */
+            tf_schema_free(&node->output_schema);
+            if (entry->infer_schema(node, &current, &node->output_schema) != TF_OK) {
+                tf_schema_free(&current);
+                plan->schema_inferred = false;
+                return TF_ERROR;
+            }
+
+            /* Advance current to this node's output */
+            tf_schema_free(&current);
+            if (tf_schema_copy_checked(&current, &node->output_schema) != TF_OK) {
+                plan->schema_inferred = false;
+                return TF_ERROR;
+            }
+        } else {
+            /* No schema inference available — propagate unknown */
+            tf_schema_free(&node->output_schema);
+            node->output_schema.known = false;
+            tf_schema_free(&current);
+            current.known = false;
+        }
+    }
+
+    /* Store final schema (schema before encoder, i.e., last transform's output) */
+    tf_schema_free(&plan->final_schema);
+    if (plan->n_nodes >= 2) {
+        /* The schema before the encoder is the second-to-last node's output */
+        if (tf_schema_copy_checked(&plan->final_schema,
+                                   &plan->nodes[plan->n_nodes - 2].output_schema) != TF_OK) {
+            tf_schema_free(&current);
+            plan->schema_inferred = false;
+            return TF_ERROR;
+        }
+    } else {
+        plan->final_schema.known = false;
+    }
+
+    tf_schema_free(&current);
+    plan->schema_inferred = true;
+    return TF_OK;
+}

package/csrc/ir_serialize.c

@@ -0,0 +1,166 @@
+/*
+ * ir_serialize.c — IR plan ↔ JSON serialization.
+ *
+ * JSON format (the .tfp format):
+ * {
+ *   "steps": [
+ *     {"op": "codec.csv.decode", "args": {"delimiter": ","}},
+ *     {"op": "filter", "args": {"expr": "col('age') > 25"}},
+ *     {"op": "codec.csv.encode", "args": {}}
+ *   ]
+ * }
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static void set_error(char **error, const char *msg) {
+    if (error) {
+        free(*error);
+        size_t len = strlen(msg) + 1;
+        *error = malloc(len);
+        if (*error) memcpy(*error, msg, len);
+    }
+}
+
+tf_ir_plan *tf_ir_from_json(const char *json, size_t len, char **error) {
+    if (error) *error = NULL;
+
+    cJSON *root = cJSON_ParseWithLength(json, len);
+    if (!root) {
+        set_error(error, "invalid JSON in plan");
+        return NULL;
+    }
+
+    cJSON *steps_arr = cJSON_GetObjectItemCaseSensitive(root, "steps");
+    if (!cJSON_IsArray(steps_arr)) {
+        set_error(error, "plan must have a 'steps' array");
+        cJSON_Delete(root);
+        return NULL;
+    }
+
+    int n = cJSON_GetArraySize(steps_arr);
+    if (n == 0) {
+        set_error(error, "plan has no steps");
+        cJSON_Delete(root);
+        return NULL;
+    }
+
+    tf_ir_plan *plan = tf_ir_plan_create();
+    if (!plan) {
+        set_error(error, "out of memory");
+        cJSON_Delete(root);
+        return NULL;
+    }
+
+    for (int i = 0; i < n; i++) {
+        cJSON *step = cJSON_GetArrayItem(steps_arr, i);
+        cJSON *op = cJSON_GetObjectItemCaseSensitive(step, "op");
+        cJSON *args = cJSON_GetObjectItemCaseSensitive(step, "args");
+
+        if (!cJSON_IsString(op)) {
+            char buf[64];
+            snprintf(buf, sizeof(buf), "step %d missing 'op' string", i);
+            set_error(error, buf);
+            tf_ir_plan_free(plan);
+            cJSON_Delete(root);
+            return NULL;
+        }
+
+        if (tf_ir_plan_add_node(plan, op->valuestring, args) != 0) {
+            set_error(error, "out of memory adding node");
+            tf_ir_plan_free(plan);
+            cJSON_Delete(root);
+            return NULL;
+        }
+    }
+
+    cJSON_Delete(root);
+    return plan;
+}
+
+char *tf_ir_to_json(const tf_ir_plan *plan) {
+    cJSON *root = cJSON_CreateObject();
+    if (!root) return NULL;
+    cJSON *steps = cJSON_CreateArray();
+    if (!steps || tf_json_add_item(root, "steps", steps) != TF_OK) {
+        cJSON_Delete(root);
+        return NULL;
+    }
+
+    for (size_t i = 0; i < plan->n_nodes; i++) {
+        const tf_ir_node *node = &plan->nodes[i];
+        cJSON *step = cJSON_CreateObject();
+        if (!step) {
+            cJSON_Delete(root);
+            return NULL;
+        }
+        if (tf_json_add_string(step, "op", node->op) != TF_OK) {
+            cJSON_Delete(step);
+            cJSON_Delete(root);
+            return NULL;
+        }
+        if (node->args) {
+            cJSON *args = cJSON_Duplicate(node->args, 1);
+            if (args) {
+                cJSON_DeleteItemFromObjectCaseSensitive(args, TF_POLICY_VALIDATED_FILE_PATH_ARG);
+                cJSON_DeleteItemFromObjectCaseSensitive(args, TF_POLICY_VALIDATED_RULES_FILE_PATH_ARG);
+            }
+            if (!args || tf_json_add_item(step, "args", args) != TF_OK) {
+                cJSON_Delete(step);
+                cJSON_Delete(root);
+                return NULL;
+            }
+        } else {
+            cJSON *args = cJSON_CreateObject();
+            if (!args || tf_json_add_item(step, "args", args) != TF_OK) {
+                cJSON_Delete(step);
+                cJSON_Delete(root);
+                return NULL;
+            }
+        }
+        if (tf_json_add_string(step, "memory_class", tf_memory_class_name(node->memory_class)) != TF_OK ||
+            tf_json_add_string(step, "emit_class", tf_emit_class_name(node->emit_class)) != TF_OK ||
+            tf_json_add_string(step, "schema_class", tf_schema_class_name(node->schema_class)) != TF_OK ||
+            tf_json_add_string(step, "state_estimate",
+                               node->state_estimate ? node->state_estimate : "unknown") != TF_OK) {
+            cJSON_Delete(step);
+            cJSON_Delete(root);
+            return NULL;
+        }
+        size_t state_bytes = 0;
+        char state_reason[192] = {0};
+        if (tf_estimate_step_state_bytes(node, &state_bytes, state_reason, sizeof(state_reason))) {
+            if (tf_json_add_number(step, "state_bytes_estimate", (double)state_bytes) != TF_OK) {
+                cJSON_Delete(step);
+                cJSON_Delete(root);
+                return NULL;
+            }
+        } else {
+            if (tf_json_add_null(step, "state_bytes_estimate") != TF_OK) {
+                cJSON_Delete(step);
+                cJSON_Delete(root);
+                return NULL;
+            }
+            if (node->memory_class == TF_MEM_KEY_STATE || node->memory_class == TF_MEM_BLOCKING) {
+                if (tf_json_add_string(step, "state_bytes_reason",
+                                       state_reason[0] ? state_reason : "no native byte estimator") != TF_OK) {
+                    cJSON_Delete(step);
+                    cJSON_Delete(root);
+                    return NULL;
+                }
+            }
+        }
+        if (tf_json_add_array_item(steps, step) != TF_OK) {
+            cJSON_Delete(root);
+            return NULL;
+        }
+    }
+
+    char *out = cJSON_PrintUnformatted(root);
+    cJSON_Delete(root);
+    return out;
+}