tranfi 0.0.2 → 0.1.2

package/csrc/wasm_api.c

@@ -0,0 +1,157 @@
+/*
+ * wasm_api.c — Emscripten WASM exports for Tranfi.
+ *
+ * Handle-based API: JS gets integer handles instead of raw pointers.
+ * Same pattern as statsim/compiler.
+ */
+
+#ifdef __EMSCRIPTEN__
+#include <emscripten.h>
+#endif
+
+#include "tranfi.h"
+#include "dsl.h"
+#include "recipes.h"
+#include <stdlib.h>
+#include <string.h>
+
+/* Handle map: slot index → tf_pipeline* */
+#define MAX_HANDLES 256
+static tf_pipeline *handles[MAX_HANDLES] = {0};
+
+static int alloc_handle(tf_pipeline *p) {
+    for (int i = 1; i < MAX_HANDLES; i++) {
+        if (!handles[i]) {
+            handles[i] = p;
+            return i;
+        }
+    }
+    return -1; /* no free slots */
+}
+
+static tf_pipeline *get_handle(int h) {
+    if (h <= 0 || h >= MAX_HANDLES) return NULL;
+    return handles[h];
+}
+
+#ifdef __EMSCRIPTEN__
+#define EXPORT EMSCRIPTEN_KEEPALIVE
+#else
+#define EXPORT
+#endif
+
+EXPORT
+int wasm_pipeline_create(const char *json, int len) {
+    tf_pipeline *p = tf_pipeline_create(json, (size_t)len);
+    if (!p) return -1;
+    int h = alloc_handle(p);
+    if (h < 0) {
+        tf_pipeline_free(p);
+        return -1;
+    }
+    return h;
+}
+
+EXPORT
+int wasm_pipeline_push(int handle, const uint8_t *data, int len) {
+    tf_pipeline *p = get_handle(handle);
+    if (!p) return -1;
+    return tf_pipeline_push(p, data, (size_t)len);
+}
+
+EXPORT
+int wasm_pipeline_finish(int handle) {
+    tf_pipeline *p = get_handle(handle);
+    if (!p) return -1;
+    return tf_pipeline_finish(p);
+}
+
+EXPORT
+int wasm_pipeline_pull(int handle, int channel, uint8_t *buf, int buf_len) {
+    tf_pipeline *p = get_handle(handle);
+    if (!p) return 0;
+    return (int)tf_pipeline_pull(p, channel, buf, (size_t)buf_len);
+}
+
+EXPORT
+const char *wasm_pipeline_error(int handle) {
+    tf_pipeline *p = get_handle(handle);
+    if (!p) return tf_last_error();
+    const char *err = tf_pipeline_error(p);
+    return err ? err : tf_last_error();
+}
+
+EXPORT
+void wasm_pipeline_free(int handle) {
+    tf_pipeline *p = get_handle(handle);
+    if (p) {
+        tf_pipeline_free(p);
+        handles[handle] = NULL;
+    }
+}
+
+/*
+ * wasm_compile_dsl — parse DSL string, return plan JSON.
+ *
+ * Returns heap-allocated JSON string (caller must free), or NULL on error.
+ * On error, wasm_pipeline_error(-1) returns the message.
+ */
+EXPORT
+char *wasm_compile_dsl(const char *dsl, int len) {
+    char *error = NULL;
+    tf_ir_plan *plan = tf_dsl_parse(dsl, (size_t)len, &error);
+    if (!plan) {
+        /* Store error so JS can retrieve it via wasm_pipeline_error(-1) */
+        (void)error; /* error accessible via tf_last_error() */
+        free(error);
+        return NULL;
+    }
+    char *json = tf_ir_plan_to_json(plan);
+    tf_ir_plan_destroy(plan);
+    return json;
+}
+
+/*
+ * wasm_compile_to_sql — compile DSL to SQL query string.
+ *
+ * Returns heap-allocated SQL string (caller must free), or NULL on error.
+ */
+EXPORT
+char *wasm_compile_to_sql(const char *dsl, int len) {
+    char *error = NULL;
+    char *sql = tf_compile_to_sql(dsl, (size_t)len, &error);
+    free(error);
+    return sql;
+}
+
+EXPORT
+const char *wasm_version(void) {
+    return tf_version();
+}
+
+/* ---- Recipe API ---- */
+
+EXPORT
+int wasm_recipe_count(void) {
+    return (int)tf_recipe_count();
+}
+
+EXPORT
+const char *wasm_recipe_name(int index) {
+    return tf_recipe_name((size_t)index);
+}
+
+EXPORT
+const char *wasm_recipe_dsl(int index) {
+    return tf_recipe_dsl((size_t)index);
+}
+
+EXPORT
+const char *wasm_recipe_description(int index) {
+    return tf_recipe_description((size_t)index);
+}
+
+EXPORT
+const char *wasm_recipe_find_dsl(const char *name) {
+    return tf_recipe_find_dsl(name);
+}

package/napi_api.c

@@ -0,0 +1,326 @@
+/*
+ * napi_api.c — Node.js N-API addon for Tranfi.
+ *
+ * Exposes the C pipeline API as native Node functions.
+ */
+
+#include <node_api.h>
+#include "tranfi.h"
+#include "dsl.h"
+#include "recipes.h"
+#include <stdlib.h>
+#include <string.h>
+
+#define NAPI_CALL(env, call)                                    \
+    do {                                                        \
+        napi_status status = (call);                            \
+        if (status != napi_ok) {                                \
+            napi_throw_error(env, NULL, "N-API call failed");   \
+            return NULL;                                        \
+        }                                                       \
+    } while (0)
+
+/*ాcreate_pipeline(planJson: string) → External */
+static napi_value napi_create_pipeline(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    if (argc < 1) {
+        napi_throw_error(env, NULL, "createPipeline requires plan JSON string");
+        return NULL;
+    }
+
+    size_t str_len;
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], NULL, 0, &str_len));
+    char *json = malloc(str_len + 1);
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], json, str_len + 1, &str_len));
+
+    tf_pipeline *p = tf_pipeline_create(json, str_len);
+    free(json);
+
+    if (!p) {
+        const char *err = tf_last_error();
+        napi_throw_error(env, NULL, err ? err : "failed to create pipeline");
+        return NULL;
+    }
+
+    napi_value external;
+    NAPI_CALL(env, napi_create_external(env, p, NULL, NULL, &external));
+    return external;
+}
+
+/* push(pipeline: External, data: Buffer) → void */
+static napi_value napi_push(napi_env env, napi_callback_info info) {
+    size_t argc = 2;
+    napi_value argv[2];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    tf_pipeline *p;
+    NAPI_CALL(env, napi_get_value_external(env, argv[0], (void **)&p));
+
+    uint8_t *data;
+    size_t len;
+    NAPI_CALL(env, napi_get_buffer_info(env, argv[1], (void **)&data, &len));
+
+    int rc = tf_pipeline_push(p, data, len);
+    if (rc != 0) {
+        const char *err = tf_pipeline_error(p);
+        napi_throw_error(env, NULL, err ? err : "push failed");
+    }
+    return NULL;
+}
+
+/* finish(pipeline: External) → void */
+static napi_value napi_finish(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    tf_pipeline *p;
+    NAPI_CALL(env, napi_get_value_external(env, argv[0], (void **)&p));
+
+    int rc = tf_pipeline_finish(p);
+    if (rc != 0) {
+        const char *err = tf_pipeline_error(p);
+        napi_throw_error(env, NULL, err ? err : "finish failed");
+    }
+    return NULL;
+}
+
+/* pull(pipeline: External, channel: number) → Buffer */
+static napi_value napi_pull(napi_env env, napi_callback_info info) {
+    size_t argc = 2;
+    napi_value argv[2];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    tf_pipeline *p;
+    NAPI_CALL(env, napi_get_value_external(env, argv[0], (void **)&p));
+
+    int32_t channel;
+    NAPI_CALL(env, napi_get_value_int32(env, argv[1], &channel));
+
+    /* Pull all available data */
+    size_t total = 0;
+    size_t cap = 65536;
+    uint8_t *result = malloc(cap);
+    uint8_t buf[65536];
+
+    for (;;) {
+        size_t n = tf_pipeline_pull(p, channel, buf, sizeof(buf));
+        if (n == 0) break;
+        if (total + n > cap) {
+            while (cap < total + n) cap *= 2;
+            result = realloc(result, cap);
+        }
+        memcpy(result + total, buf, n);
+        total += n;
+    }
+
+    napi_value buffer;
+    void *buf_data;
+    NAPI_CALL(env, napi_create_buffer_copy(env, total, result, &buf_data, &buffer));
+    free(result);
+    return buffer;
+}
+
+/* free(pipeline: External) → void */
+static napi_value napi_free_pipeline(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    tf_pipeline *p;
+    NAPI_CALL(env, napi_get_value_external(env, argv[0], (void **)&p));
+
+    tf_pipeline_free(p);
+    return NULL;
+}
+
+/* version() → string */
+static napi_value napi_tranfi_version(napi_env env, napi_callback_info info) {
+    (void)info;
+    const char *v = tf_version();
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, v, strlen(v), &result));
+    return result;
+}
+
+/* error(pipeline: External) → string | null */
+static napi_value napi_tranfi_error(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    tf_pipeline *p;
+    NAPI_CALL(env, napi_get_value_external(env, argv[0], (void **)&p));
+
+    const char *err = tf_pipeline_error(p);
+    if (!err) {
+        napi_value null_val;
+        napi_get_null(env, &null_val);
+        return null_val;
+    }
+
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, err, strlen(err), &result));
+    return result;
+}
+
+/* compileDsl(dslString: string) → string (plan JSON) */
+static napi_value napi_compile_dsl(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    if (argc < 1) {
+        napi_throw_error(env, NULL, "compileDsl requires DSL string");
+        return NULL;
+    }
+
+    size_t str_len;
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], NULL, 0, &str_len));
+    char *dsl = malloc(str_len + 1);
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], dsl, str_len + 1, &str_len));
+
+    char *error = NULL;
+    tf_ir_plan *plan = tf_dsl_parse(dsl, str_len, &error);
+    free(dsl);
+
+    if (!plan) {
+        napi_throw_error(env, NULL, error ? error : "DSL parse failed");
+        free(error);
+        return NULL;
+    }
+
+    char *json = tf_ir_plan_to_json(plan);
+    tf_ir_plan_destroy(plan);
+
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, json, strlen(json), &result));
+    free(json);
+    return result;
+}
+
+/* compileToSql(dslString: string) → string (SQL query) */
+static napi_value napi_compile_to_sql(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+
+    if (argc < 1) {
+        napi_throw_error(env, NULL, "compileToSql requires DSL string");
+        return NULL;
+    }
+
+    size_t str_len;
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], NULL, 0, &str_len));
+    char *dsl = malloc(str_len + 1);
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], dsl, str_len + 1, &str_len));
+
+    char *error = NULL;
+    char *sql = tf_compile_to_sql(dsl, str_len, &error);
+    free(dsl);
+
+    if (!sql) {
+        napi_throw_error(env, NULL, error ? error : "SQL compilation failed");
+        free(error);
+        return NULL;
+    }
+
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, sql, strlen(sql), &result));
+    tf_string_free(sql);
+    return result;
+}
+
+/* recipeCount() → number */
+static napi_value napi_recipe_count(napi_env env, napi_callback_info info) {
+    (void)info;
+    napi_value result;
+    NAPI_CALL(env, napi_create_uint32(env, (uint32_t)tf_recipe_count(), &result));
+    return result;
+}
+
+/* recipeName(index: number) → string | null */
+static napi_value napi_recipe_name(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+    uint32_t index;
+    NAPI_CALL(env, napi_get_value_uint32(env, argv[0], &index));
+    const char *s = tf_recipe_name(index);
+    if (!s) { napi_value n; napi_get_null(env, &n); return n; }
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, s, strlen(s), &result));
+    return result;
+}
+
+/* recipeDsl(index: number) → string | null */
+static napi_value napi_recipe_dsl(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+    uint32_t index;
+    NAPI_CALL(env, napi_get_value_uint32(env, argv[0], &index));
+    const char *s = tf_recipe_dsl(index);
+    if (!s) { napi_value n; napi_get_null(env, &n); return n; }
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, s, strlen(s), &result));
+    return result;
+}
+
+/* recipeDescription(index: number) → string | null */
+static napi_value napi_recipe_description(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+    uint32_t index;
+    NAPI_CALL(env, napi_get_value_uint32(env, argv[0], &index));
+    const char *s = tf_recipe_description(index);
+    if (!s) { napi_value n; napi_get_null(env, &n); return n; }
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, s, strlen(s), &result));
+    return result;
+}
+
+/* recipeFindDsl(name: string) → string | null */
+static napi_value napi_recipe_find_dsl(napi_env env, napi_callback_info info) {
+    size_t argc = 1;
+    napi_value argv[1];
+    NAPI_CALL(env, napi_get_cb_info(env, info, &argc, argv, NULL, NULL));
+    size_t str_len;
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], NULL, 0, &str_len));
+    char *name = malloc(str_len + 1);
+    NAPI_CALL(env, napi_get_value_string_utf8(env, argv[0], name, str_len + 1, &str_len));
+    const char *dsl = tf_recipe_find_dsl(name);
+    free(name);
+    if (!dsl) { napi_value n; napi_get_null(env, &n); return n; }
+    napi_value result;
+    NAPI_CALL(env, napi_create_string_utf8(env, dsl, strlen(dsl), &result));
+    return result;
+}
+
+/* Module init */
+static napi_value init(napi_env env, napi_value exports) {
+    napi_property_descriptor props[] = {
+        {"createPipeline",    NULL, napi_create_pipeline,    NULL, NULL, NULL, napi_default, NULL},
+        {"push",              NULL, napi_push,               NULL, NULL, NULL, napi_default, NULL},
+        {"finish",            NULL, napi_finish,             NULL, NULL, NULL, napi_default, NULL},
+        {"pull",              NULL, napi_pull,               NULL, NULL, NULL, napi_default, NULL},
+        {"free",              NULL, napi_free_pipeline,      NULL, NULL, NULL, napi_default, NULL},
+        {"version",           NULL, napi_tranfi_version,     NULL, NULL, NULL, napi_default, NULL},
+        {"error",             NULL, napi_tranfi_error,       NULL, NULL, NULL, napi_default, NULL},
+        {"compileDsl",        NULL, napi_compile_dsl,        NULL, NULL, NULL, napi_default, NULL},
+        {"compileToSql",      NULL, napi_compile_to_sql,     NULL, NULL, NULL, napi_default, NULL},
+        {"recipeCount",       NULL, napi_recipe_count,       NULL, NULL, NULL, napi_default, NULL},
+        {"recipeName",        NULL, napi_recipe_name,        NULL, NULL, NULL, napi_default, NULL},
+        {"recipeDsl",         NULL, napi_recipe_dsl,         NULL, NULL, NULL, napi_default, NULL},
+        {"recipeDescription", NULL, napi_recipe_description, NULL, NULL, NULL, napi_default, NULL},
+        {"recipeFindDsl",     NULL, napi_recipe_find_dsl,    NULL, NULL, NULL, napi_default, NULL},
+    };
+    NAPI_CALL(env, napi_define_properties(env, exports, 14, props));
+    return exports;
+}
+
+NAPI_MODULE(NODE_GYP_MODULE_NAME, init)

package/package.json

@@ -1,68 +1,57 @@
 {
   "name": "tranfi",
-  "version": "0.0.2",
-  "description": "Stream-based tool for file processing",
-  "main": "src/main.js",
+  "version": "0.1.2",
+  "description": "Streaming ETL language + runtime",
+  "type": "module",
+  "main": "src/index.js",
+  "bin": {
+    "tranfi": "src/cli.js"
+  },
+  "exports": {
+    ".": "./src/index.js",
+    "./wasm": "./wasm/index.js"
+  },
+  "files": [
+    "src/",
+    "wasm/",
+    "csrc/",
+    "app/",
+    "napi_api.c",
+    "binding.gyp",
+    "README.md"
+  ],
   "scripts": {
-    "build-dev": "webpack --mode=development --progress --stats-children --env DEVELOPMENT",
-    "build": "webpack --mode=production --progress && webpack --mode=production --progress --env RUNTIME",
-    "watch": "nodemon --watch . --ignore dist --ext vue,js,css,html --exec 'npm run build-dev'",
-    "prepublishOnly": "npm run build",
-    "test": "jest test.js --detectOpenHandles",
-    "test-head": "HEADLESS=false npm test"
+    "prepack": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && rm -rf app && cp -r ../app/dist app && rm -rf app/wasm app/lib",
+    "install": "node-gyp rebuild 2>/dev/null || true",
+    "build:native": "rm -rf csrc && mkdir -p csrc && cp ../src/*.c ../src/*.h csrc/ && node-gyp rebuild",
+    "test": "node ../test/test_node.js"
+  },
+  "peerDependencies": {
+    "duckdb": ">=1.0.0",
+    "@duckdb/duckdb-wasm": ">=1.0.0"
   },
-  "author": "Anton Zemlyansky",
+  "peerDependenciesMeta": {
+    "duckdb": {
+      "optional": true
+    },
+    "@duckdb/duckdb-wasm": {
+      "optional": true
+    }
+  },
+  "keywords": [
+    "etl",
+    "csv",
+    "streaming",
+    "data",
+    "pipeline",
+    "transform"
+  ],
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/tranfi/tranfi"
-  },
-  "dependencies": {
-    "@jseeio/jsee": "^0.2.8",
-    "buffer": "^6.0.3",
-    "bulma": "^0.9.3",
-    "csv-parse": "^4.16.3",
-    "element-plus": "^1.2.0-beta.6",
-    "equal-vue": "^0.79.3",
-    "file-saver": "^2.0.2",
-    "filestream": "^5.0.0",
-    "filtrex": "^2.2.3",
-    "JSONStream": "^1.3.5",
-    "notyf": "^3.10.0",
-    "process": "^0.11.10",
-    "stream-browserify": "^3.0.0",
-    "stream-http": "^3.2.0",
-    "through2": "^4.0.2",
-    "unzip-stream": "^0.3.1",
-    "url": "^0.11.0",
-    "vue": "^3.2.23",
-    "vue-style-loader": "^4.1.3",
-    "vue3-json-viewer": "^1.0.4",
-    "vuedraggable": "^4.1.0",
-    "vuex": "^4.0.2"
+    "url": "https://github.com/tranfi/tranfi"
   },
   "devDependencies": {
-    "@babel/core": "^7.16.5",
-    "@babel/plugin-transform-runtime": "^7.4.4",
-    "@babel/preset-env": "^7.16.5",
-    "babel-loader": "^8.2.3",
-    "css-loader": "^6.5.1",
-    "http-server": "^14.0.0",
-    "jest": "^27.4.5",
-    "jest-puppeteer": "^6.0.3",
-    "node-sass": "^7.0.0",
-    "nodemon": "^2.0.15",
-    "puppeteer": "^1.20.0",
-    "sass-loader": "^12.4.0",
-    "source-map-loader": "^3.0.0",
-    "style-loader": "^3.3.1",
-    "tape": "^4.9.1",
-    "terser-webpack-plugin": "^5.3.0",
-    "uglify-es": "^3.3.9",
-    "vue-loader": "^17.0.0",
-    "vue-template-compiler": "^2.6.14",
-    "webpack": "^5.65.0",
-    "webpack-cli": "^4.9.1",
-    "worker-loader": "^3.0.8"
+    "duckdb": "^1.4.4"
   }
 }