tranfi 0.0.2 → 0.1.2

package/csrc/codec_text.c

@@ -0,0 +1,229 @@
+/*
+ * codec_text.c — Plain text line codec (newline-split).
+ *
+ * Decoder: splits on newlines, each line → one row in single "_line" column.
+ *   - No type detection, no field parsing — just memchr for newlines.
+ *   - Emits batch at batch_size.
+ *
+ * Encoder: writes _line string + \n per row.
+ *   - Fallback: if no _line column, concatenate all string columns with tab.
+ */
+
+#include "internal.h"
+#include "cJSON.h"
+#include <stdlib.h>
+#include <string.h>
+
+#define DEFAULT_BATCH_SIZE 1024
+
+/* ================================================================
+ * Text Decoder
+ * ================================================================ */
+
+typedef struct {
+    size_t    batch_size;
+    tf_buffer line_buf;
+    tf_batch *batch;
+    size_t    rows_buffered;
+} text_decoder_state;
+
+static tf_batch *make_text_batch(size_t capacity) {
+    tf_batch *b = tf_batch_create(1, capacity);
+    if (!b) return NULL;
+    tf_batch_set_schema(b, 0, "_line", TF_TYPE_STRING);
+    return b;
+}
+
+static int text_decode(tf_decoder *self, const uint8_t *data, size_t len,
+                       tf_batch ***out, size_t *n_out) {
+    text_decoder_state *st = self->state;
+    *out = NULL;
+    *n_out = 0;
+
+    if (tf_buffer_write(&st->line_buf, data, len) != TF_OK) return TF_ERROR;
+
+    size_t out_cap = 0;
+    uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+    size_t buf_len = st->line_buf.len - st->line_buf.read_pos;
+
+    size_t line_start = 0;
+    for (size_t i = 0; i < buf_len; i++) {
+        if (buf[i] == '\n') {
+            size_t line_len = i - line_start;
+            /* Strip trailing \r for CRLF */
+            if (line_len > 0 && buf[line_start + line_len - 1] == '\r')
+                line_len--;
+
+            if (!st->batch) {
+                st->batch = make_text_batch(st->batch_size);
+                if (!st->batch) return TF_ERROR;
+            }
+
+            size_t row = st->batch->n_rows;
+            if (tf_batch_ensure_capacity(st->batch, row + 1) != TF_OK)
+                return TF_ERROR;
+
+            /* Copy line into batch */
+            char *line_str = malloc(line_len + 1);
+            if (!line_str) return TF_ERROR;
+            memcpy(line_str, buf + line_start, line_len);
+            line_str[line_len] = '\0';
+            tf_batch_set_string(st->batch, row, 0, line_str);
+            free(line_str);
+            st->batch->n_rows = row + 1;
+            st->rows_buffered++;
+
+            line_start = i + 1;
+
+            /* Emit batch if full */
+            if (st->rows_buffered >= st->batch_size) {
+                if (*n_out >= out_cap) {
+                    out_cap = (out_cap == 0) ? 4 : out_cap * 2;
+                    *out = realloc(*out, out_cap * sizeof(tf_batch *));
+                }
+                (*out)[(*n_out)++] = st->batch;
+                st->batch = NULL;
+                st->rows_buffered = 0;
+            }
+        }
+    }
+
+    st->line_buf.read_pos += line_start;
+    tf_buffer_compact(&st->line_buf);
+    return TF_OK;
+}
+
+static int text_flush(tf_decoder *self, tf_batch ***out, size_t *n_out) {
+    text_decoder_state *st = self->state;
+    *out = NULL;
+    *n_out = 0;
+    size_t out_cap = 0;
+
+    /* Process remaining data as a final line */
+    size_t remaining = tf_buffer_readable(&st->line_buf);
+    if (remaining > 0) {
+        uint8_t *buf = st->line_buf.data + st->line_buf.read_pos;
+
+        if (!st->batch) {
+            st->batch = make_text_batch(st->batch_size);
+            if (!st->batch) return TF_ERROR;
+        }
+
+        size_t row = st->batch->n_rows;
+        if (tf_batch_ensure_capacity(st->batch, row + 1) != TF_OK)
+            return TF_ERROR;
+
+        /* Strip trailing \r */
+        size_t line_len = remaining;
+        if (line_len > 0 && buf[line_len - 1] == '\r')
+            line_len--;
+
+        char *line_str = malloc(line_len + 1);
+        if (!line_str) return TF_ERROR;
+        memcpy(line_str, buf, line_len);
+        line_str[line_len] = '\0';
+        tf_batch_set_string(st->batch, row, 0, line_str);
+        free(line_str);
+        st->batch->n_rows = row + 1;
+        st->rows_buffered++;
+
+        st->line_buf.read_pos = st->line_buf.len;
+    }
+
+    /* Emit remaining batch */
+    if (st->batch && st->rows_buffered > 0) {
+        if (*n_out >= out_cap) {
+            out_cap = (*n_out == 0) ? 1 : out_cap * 2;
+            *out = realloc(*out, out_cap * sizeof(tf_batch *));
+        }
+        (*out)[(*n_out)++] = st->batch;
+        st->batch = NULL;
+        st->rows_buffered = 0;
+    }
+
+    return TF_OK;
+}
+
+static void text_decoder_destroy(tf_decoder *self) {
+    text_decoder_state *st = self->state;
+    if (st) {
+        tf_buffer_free(&st->line_buf);
+        if (st->batch) tf_batch_free(st->batch);
+        free(st);
+    }
+    free(self);
+}
+
+tf_decoder *tf_text_decoder_create(const cJSON *args) {
+    text_decoder_state *st = calloc(1, sizeof(text_decoder_state));
+    if (!st) return NULL;
+
+    st->batch_size = DEFAULT_BATCH_SIZE;
+    if (args) {
+        cJSON *bs = cJSON_GetObjectItemCaseSensitive(args, "batch_size");
+        if (cJSON_IsNumber(bs) && bs->valueint > 0)
+            st->batch_size = (size_t)bs->valueint;
+    }
+
+    tf_buffer_init(&st->line_buf);
+
+    tf_decoder *dec = malloc(sizeof(tf_decoder));
+    if (!dec) { free(st); return NULL; }
+    dec->decode = text_decode;
+    dec->flush = text_flush;
+    dec->destroy = text_decoder_destroy;
+    dec->state = st;
+    return dec;
+}
+
+/* ================================================================
+ * Text Encoder
+ * ================================================================ */
+
+static int text_encode(tf_encoder *self, tf_batch *in, tf_buffer *out) {
+    (void)self;
+
+    /* Find _line column index */
+    int line_col = tf_batch_col_index(in, "_line");
+
+    for (size_t r = 0; r < in->n_rows; r++) {
+        if (line_col >= 0) {
+            /* Write _line column value */
+            if (!tf_batch_is_null(in, r, (size_t)line_col)) {
+                const char *s = tf_batch_get_string(in, r, (size_t)line_col);
+                tf_buffer_write(out, (const uint8_t *)s, strlen(s));
+            }
+        } else {
+            /* Fallback: concatenate all string columns with tab */
+            for (size_t c = 0; c < in->n_cols; c++) {
+                if (c > 0) tf_buffer_write(out, (const uint8_t *)"\t", 1);
+                if (!tf_batch_is_null(in, r, c) && in->col_types[c] == TF_TYPE_STRING) {
+                    const char *s = tf_batch_get_string(in, r, c);
+                    tf_buffer_write(out, (const uint8_t *)s, strlen(s));
+                }
+            }
+        }
+        tf_buffer_write(out, (const uint8_t *)"\n", 1);
+    }
+    return TF_OK;
+}
+
+static int text_encoder_flush(tf_encoder *self, tf_buffer *out) {
+    (void)self; (void)out;
+    return TF_OK;
+}
+
+static void text_encoder_destroy(tf_encoder *self) {
+    free(self);
+}
+
+tf_encoder *tf_text_encoder_create(const cJSON *args) {
+    (void)args;
+    tf_encoder *enc = malloc(sizeof(tf_encoder));
+    if (!enc) return NULL;
+    enc->encode = text_encode;
+    enc->flush = text_encoder_flush;
+    enc->destroy = text_encoder_destroy;
+    enc->state = NULL;
+    return enc;
+}

package/csrc/compiler.c

@@ -0,0 +1,102 @@
+/*
+ * compiler.c — L2 IR → L1 native target compilation.
+ *
+ * Iterates IR nodes, looks up each op in the registry,
+ * and calls create_native() to build live decoder/steps/encoder structs.
+ * Replaces the constructor dispatch loop that was in plan.c.
+ */
+
+#include "ir.h"
+#include "internal.h"
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+static void set_error(char **error, const char *msg) {
+    if (error) {
+        free(*error);
+        size_t len = strlen(msg) + 1;
+        *error = malloc(len);
+        if (*error) memcpy(*error, msg, len);
+    }
+}
+
+int tf_compile_native(const tf_ir_plan *plan,
+                      tf_decoder **out_decoder, tf_step ***out_steps,
+                      size_t *out_n_steps, tf_encoder **out_encoder,
+                      char **error) {
+    *out_decoder = NULL;
+    *out_steps = NULL;
+    *out_n_steps = 0;
+    *out_encoder = NULL;
+    if (error) *error = NULL;
+
+    /* Allocate steps array (max = n_nodes, since some are decoder/encoder) */
+    tf_step **steps = calloc(plan->n_nodes, sizeof(tf_step *));
+    if (!steps) {
+        set_error(error, "out of memory");
+        return TF_ERROR;
+    }
+    size_t n_steps = 0;
+    tf_decoder *decoder = NULL;
+    tf_encoder *encoder = NULL;
+
+    for (size_t i = 0; i < plan->n_nodes; i++) {
+        const tf_ir_node *node = &plan->nodes[i];
+        const tf_op_entry *entry = tf_op_registry_find(node->op);
+
+        if (!entry) {
+            char buf[256];
+            snprintf(buf, sizeof(buf), "unknown op: '%s'", node->op);
+            set_error(error, buf);
+            goto fail;
+        }
+
+        if (!entry->create_native) {
+            if (entry->tier == TF_TIER_ECOSYSTEM) {
+                /* Ecosystem stubs are skipped — host must handle */
+                char buf[256];
+                snprintf(buf, sizeof(buf), "op '%s' has no native target", node->op);
+                set_error(error, buf);
+                goto fail;
+            }
+            /* Core ops with NULL constructor are no-ops (e.g. flatten) */
+            continue;
+        }
+
+        void *obj = entry->create_native(node->args);
+        if (!obj) {
+            char buf[256];
+            snprintf(buf, sizeof(buf), "failed to create '%s'", node->op);
+            set_error(error, buf);
+            goto fail;
+        }
+
+        switch (entry->kind) {
+        case TF_OP_DECODER:
+            decoder = (tf_decoder *)obj;
+            break;
+        case TF_OP_ENCODER:
+            encoder = (tf_encoder *)obj;
+            break;
+        case TF_OP_TRANSFORM:
+            steps[n_steps++] = (tf_step *)obj;
+            break;
+        }
+    }
+
+    *out_decoder = decoder;
+    *out_steps = steps;
+    *out_n_steps = n_steps;
+    *out_encoder = encoder;
+    return TF_OK;
+
+fail:
+    if (decoder) decoder->destroy(decoder);
+    if (encoder) encoder->destroy(encoder);
+    for (size_t i = 0; i < n_steps; i++) {
+        if (steps[i]) steps[i]->destroy(steps[i]);
+    }
+    free(steps);
+    return TF_ERROR;
+}

package/csrc/date_utils.h

@@ -0,0 +1,94 @@
+/*
+ * date_utils.h — Date/timestamp conversion utilities (header-only).
+ *
+ * Uses Howard Hinnant's civil date algorithms (public domain).
+ * Epoch: 1970-01-01.  Date = int32_t days.  Timestamp = int64_t microseconds.
+ */
+
+#ifndef TF_DATE_UTILS_H
+#define TF_DATE_UTILS_H
+
+#include <stdint.h>
+#include <stdio.h>
+
+/* Days from civil date to epoch (1970-01-01 = 0). */
+static inline int32_t tf_date_from_ymd(int y, int m, int d) {
+    y -= (m <= 2);
+    int era = (y >= 0 ? y : y - 399) / 400;
+    unsigned yoe = (unsigned)(y - era * 400);
+    unsigned doy = (153 * (m > 2 ? m - 3 : m + 9) + 2) / 5 + (unsigned)d - 1;
+    unsigned doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
+    return era * 146097 + (int)doe - 719468;
+}
+
+/* Epoch days back to civil date. */
+static inline void tf_date_to_ymd(int32_t days, int *y, int *m, int *d) {
+    days += 719468;
+    int era = (days >= 0 ? days : days - 146096) / 146097;
+    unsigned doe = (unsigned)(days - era * 146097);
+    unsigned yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
+    *y = (int)yoe + era * 400;
+    unsigned doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+    unsigned mp = (5 * doy + 2) / 153;
+    *d = (int)(doy - (153 * mp + 2) / 5 + 1);
+    *m = (int)(mp < 10 ? mp + 3 : mp - 9);
+    *y += (*m <= 2);
+}
+
+/* Microseconds since epoch to date/time components. */
+static inline void tf_timestamp_to_parts(int64_t us, int *y, int *mo, int *d,
+                                          int *h, int *mi, int *s, int *frac_us) {
+    int64_t day_us = 86400LL * 1000000LL;
+    int32_t days = (int32_t)(us / day_us);
+    int64_t rem = us % day_us;
+    if (rem < 0) { days--; rem += day_us; }
+    tf_date_to_ymd(days, y, mo, d);
+    *frac_us = (int)(rem % 1000000);
+    rem /= 1000000;
+    *s = (int)(rem % 60); rem /= 60;
+    *mi = (int)(rem % 60); rem /= 60;
+    *h = (int)rem;
+}
+
+/* Date/time components to microseconds since epoch. */
+static inline int64_t tf_timestamp_from_parts(int y, int mo, int d,
+                                               int h, int mi, int s, int frac_us) {
+    int32_t days = tf_date_from_ymd(y, mo, d);
+    return (int64_t)days * 86400LL * 1000000LL
+         + (int64_t)h * 3600000000LL
+         + (int64_t)mi * 60000000LL
+         + (int64_t)s * 1000000LL
+         + frac_us;
+}
+
+/* Weekday: 0=Sunday..6=Saturday. */
+static inline int tf_date_weekday(int32_t days) {
+    return days >= -4 ? (days + 4) % 7 : (days + 5) % 7 + 6;
+}
+
+/* Format date as YYYY-MM-DD into buf (must be >= 11 bytes). Returns buf. */
+static inline char *tf_date_format(int32_t days, char *buf, size_t buflen) {
+    int y, m, d;
+    tf_date_to_ymd(days, &y, &m, &d);
+    snprintf(buf, buflen, "%04d-%02d-%02d", y, m, d);
+    return buf;
+}
+
+/* Format timestamp as ISO 8601 into buf (must be >= 32 bytes). Returns buf. */
+static inline char *tf_timestamp_format(int64_t us, char *buf, size_t buflen) {
+    int y, mo, d, h, mi, s, frac;
+    tf_timestamp_to_parts(us, &y, &mo, &d, &h, &mi, &s, &frac);
+    if (frac > 0) {
+        int len = snprintf(buf, buflen, "%04d-%02d-%02dT%02d:%02d:%02d.%06d",
+                           y, mo, d, h, mi, s, frac);
+        /* Trim trailing zeros */
+        while (len > 0 && buf[len - 1] == '0') len--;
+        buf[len++] = 'Z';
+        buf[len] = '\0';
+    } else {
+        snprintf(buf, buflen, "%04d-%02d-%02dT%02d:%02d:%02dZ", y, mo, d, h, mi, s);
+    }
+    return buf;
+}
+
+#endif /* TF_DATE_UTILS_H */