tranfi 0.1.2 → 0.2.0

package/csrc/tranfi.h

@@ -9,8 +9,11 @@
 #ifndef TRANFI_H
 #define TRANFI_H
 
+#include "config.h"
 #include <stddef.h>
 #include <stdint.h>
+#include <stdio.h>
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -25,10 +28,92 @@ extern "C" {
 
 /* Return codes */
 #define TF_OK    0
+#define TF_DONE  1
 #define TF_ERROR (-1)
 
-/* Opaque pipeline handle */
+/* Shared value types used by schemas and public batch accessors. */
+#ifndef TF_TYPE_DEFINED
+#define TF_TYPE_DEFINED
+typedef enum tf_type {
+    TF_TYPE_NULL = 0,
+    TF_TYPE_BOOL,
+    TF_TYPE_INT64,
+    TF_TYPE_FLOAT64,
+    TF_TYPE_STRING,
+    TF_TYPE_DATE,       /* int32_t: days since 1970-01-01 */
+    TF_TYPE_TIMESTAMP,  /* int64_t: microseconds since 1970-01-01T00:00:00Z */
+} tf_type;
+#endif
+
+/* Opaque pipeline and columnar batch handles. */
 typedef struct tf_pipeline tf_pipeline;
+typedef struct tf_batch tf_batch;
+
+/*
+ * Sink callback used by tf_pipeline_set_sink() and tf_pipeline_drain().
+ * Return TF_OK to continue, TF_ERROR to stop the pipeline with an error.
+ * The data pointer is valid only for the duration of the callback.
+ */
+typedef int (*tf_pipeline_sink_fn)(int channel, const uint8_t *data, size_t len, void *user);
+
+/*
+ * Batch callback for embedders that consume transformed columnar batches before
+ * encoding. The batch and any returned string pointers are valid only for the
+ * duration of the callback; copy data that must outlive the callback.
+ */
+typedef int (*tf_pipeline_batch_sink_fn)(const tf_batch *batch, void *user);
+
+/* Progress snapshot passed to tf_pipeline_progress_fn. */
+typedef struct tf_pipeline_progress {
+    size_t bytes_in;
+    size_t bytes_out;
+    size_t rows_in;
+    size_t rows_out;
+    size_t batches_in;
+    size_t batches_out;
+    const char *phase; /* push, decoder_flush, step_flush, encoder_flush, stats, done */
+    int finished;
+} tf_pipeline_progress;
+
+/*
+ * Progress callback invoked at batch/flush boundaries. Return TF_OK to continue
+ * or TF_ERROR to stop the pipeline with `progress callback failed`.
+ */
+typedef int (*tf_pipeline_progress_fn)(const tf_pipeline_progress *progress, void *user);
+
+
+/*
+ * Host policy for embedded or untrusted execution surfaces.
+ * File-taking pipeline args are validated before native construction; when
+ * workspace_root or resolve_path is provided, accepted paths are rewritten to
+ * the resolved host path in the mutable IR used for execution.
+ */
+typedef int (*tf_resolve_path_fn)(const char *logical, char *out, size_t out_sz, void *user);
+
+typedef struct tf_host_policy {
+    bool allow_fs;
+    bool allow_net;
+    bool allow_spill;
+    bool allow_blocking;
+    bool allow_rules_file;
+    const char *workspace_root;
+    tf_resolve_path_fn resolve_path;
+    void *user;
+} tf_host_policy;
+
+/* Read-only columnar batch accessors for batch callbacks. */
+size_t      tf_batch_num_rows(const tf_batch *b);
+size_t      tf_batch_num_cols(const tf_batch *b);
+const char *tf_batch_col_name(const tf_batch *b, size_t col);
+tf_type     tf_batch_col_type(const tf_batch *b, size_t col);
+int         tf_batch_col_index(const tf_batch *b, const char *name);
+bool        tf_batch_is_null(const tf_batch *b, size_t row, size_t col);
+bool        tf_batch_get_bool(const tf_batch *b, size_t row, size_t col);
+int64_t     tf_batch_get_int64(const tf_batch *b, size_t row, size_t col);
+double      tf_batch_get_float64(const tf_batch *b, size_t row, size_t col);
+const char *tf_batch_get_string(const tf_batch *b, size_t row, size_t col);
+int32_t     tf_batch_get_date(const tf_batch *b, size_t row, size_t col);
+int64_t     tf_batch_get_timestamp(const tf_batch *b, size_t row, size_t col);
 
 /*
  * Create a pipeline from a JSON plan.
@@ -36,6 +121,8 @@ typedef struct tf_pipeline tf_pipeline;
  * use tf_last_error() instead).
  */
 tf_pipeline *tf_pipeline_create(const char *plan_json, size_t len);
+tf_pipeline *tf_pipeline_create_with_host_policy(const char *plan_json, size_t len,
+                                             const tf_host_policy *policy);
 
 /* Free all resources associated with a pipeline. */
 void tf_pipeline_free(tf_pipeline *p);
@@ -52,6 +139,14 @@ int tf_pipeline_push(tf_pipeline *p, const uint8_t *data, size_t len);
  */
 int tf_pipeline_finish(tf_pipeline *p);
 
+/*
+ * Advance finish processing by at most one flush boundary.
+ * Returns TF_OK while more finish work remains, TF_DONE once final stats are
+ * emitted, or TF_ERROR on failure. Hosts can call pull()/drain() between calls
+ * to avoid retaining finish-time output in the pipeline buffer.
+ */
+int tf_pipeline_finish_step(tf_pipeline *p);
+
 /*
  * Pull output bytes from a channel.
  * Writes up to buf_len bytes into buf.
@@ -59,6 +154,74 @@ int tf_pipeline_finish(tf_pipeline *p);
  */
 size_t tf_pipeline_pull(tf_pipeline *p, int channel, uint8_t *buf, size_t buf_len);
 
+/*
+ * Drain all currently buffered bytes from one channel to a callback.
+ * Returns TF_OK on success or TF_ERROR if arguments are invalid or the callback fails.
+ */
+int tf_pipeline_drain(tf_pipeline *p, int channel, tf_pipeline_sink_fn sink, void *user);
+
+/*
+ * Register or clear an automatic sink for a channel. When set, the pipeline drains
+ * that channel after each push()/finish() batch boundary, so embedders can write to
+ * FILE*, file descriptors, sockets, or host stream adapters without retaining all
+ * output in the pipeline buffer. Pass NULL as sink to clear the registered sink.
+ */
+int tf_pipeline_set_sink(tf_pipeline *p, int channel, tf_pipeline_sink_fn sink, void *user);
+
+/*
+ * Register or clear an automatic sink for transformed output batches before
+ * encoding. This is the columnar-batch counterpart to byte sinks: it is invoked
+ * at output batch boundaries during push()/finish_step()/finish(). Pass NULL as
+ * sink to clear the registered batch sink.
+ */
+int tf_pipeline_set_batch_sink(tf_pipeline *p, tf_pipeline_batch_sink_fn sink, void *user);
+
+/*
+ * Enable or disable encoded main byte output. Enabled by default. Batch-only
+ * embedders can disable it after setting a batch sink to avoid CSV/JSONL byte
+ * generation while still keeping side channels and row stats available.
+ */
+int tf_pipeline_set_encode_output(tf_pipeline *p, int enabled);
+
+/*
+ * Register or clear a progress callback. If interval_rows is 0, the callback is
+ * invoked at every batch/flush boundary; otherwise it is invoked when rows_in or
+ * rows_out advances by at least interval_rows, plus once at finish.
+ */
+int tf_pipeline_set_progress_callback(tf_pipeline *p, tf_pipeline_progress_fn cb,
+                                      void *user, size_t interval_rows);
+
+/*
+ * Set or clear the current host source name. Row-local transforms such as
+ * source-name can append this value without the core opening files itself.
+ * Pass NULL or an empty string to clear it.
+ */
+int tf_pipeline_set_source_name(tf_pipeline *p, const char *name);
+
+/*
+ * Flush only the decoder input boundary and process any decoded batches, without
+ * finishing transforms, encoders, or stats. Hosts use this between sequential
+ * input files so partial records and decoder batches keep the correct source
+ * metadata while the pipeline state remains global.
+ */
+int tf_pipeline_flush_input(tf_pipeline *p);
+
+/*
+ * Run a pipeline from an input FILE* to an output FILE* using bounded chunks.
+ * Main output is streamed to out; side channels remain available via pull().
+ * If chunk_size is 0, a 64 KiB default is used. The pipeline is finished on
+ * success and must not be pushed again.
+ */
+int tf_pipeline_run_file(tf_pipeline *p, FILE *in, FILE *out, size_t chunk_size);
+
+/*
+ * Run a pipeline from POSIX input/output file descriptors using bounded chunks.
+ * Main output is streamed to out_fd; side channels remain available via pull().
+ * If chunk_size is 0, a 64 KiB default is used. Returns TF_ERROR on
+ * non-POSIX platforms.
+ */
+int tf_pipeline_run_fd(tf_pipeline *p, int in_fd, int out_fd, size_t chunk_size);
+
 /* Get the last error message, or NULL if no error. */
 const char *tf_pipeline_error(tf_pipeline *p);
 
@@ -81,6 +244,7 @@ char *tf_ir_plan_to_json(const tf_ir_plan *plan);
 
 /* Validate an IR plan. Returns TF_OK or TF_ERROR. */
 int tf_ir_plan_validate(tf_ir_plan *plan);
+int tf_ir_plan_validate_with_host_policy(tf_ir_plan *plan, const tf_host_policy *policy);
 
 /* Infer schemas through an IR plan. Best-effort, non-fatal. */
 int tf_ir_plan_infer_schema(tf_ir_plan *plan);
@@ -90,9 +254,13 @@ void tf_ir_plan_destroy(tf_ir_plan *plan);
 
 /* Create a pipeline from a pre-built IR plan. */
 tf_pipeline *tf_pipeline_create_from_ir(const tf_ir_plan *plan);
+tf_pipeline *tf_pipeline_create_from_ir_with_host_policy(const tf_ir_plan *plan,
+                                                      const tf_host_policy *policy);
 
 /* Compile a DSL string to a JSON recipe. Caller frees with tf_string_free(). */
 char *tf_compile_dsl(const char *dsl, size_t len, char **error);
+char *tf_compile_dsl_with_host_policy(const char *dsl, size_t len,
+                                      const tf_host_policy *policy, char **error);
 
 /* Compile a DSL string directly to SQL. Caller frees with tf_string_free(). */
 char *tf_compile_to_sql(const char *dsl, size_t len, char **error);

package/csrc/transform.h

@@ -0,0 +1,209 @@
+/* Generic prepared-transform C API. */
+
+#ifndef TRANFI_TRANSFORM_H
+#define TRANFI_TRANSFORM_H
+
+#include "tranfi.h"
+#include <stddef.h>
+#include <stdint.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum tf_transform_code {
+    TF_TRANSFORM_OK = 0,
+    TF_TRANSFORM_INVALID_ARGUMENT = 100,
+    TF_TRANSFORM_INVALID_RECIPE = 101,
+    TF_TRANSFORM_SCHEMA_MISMATCH = 102,
+    TF_TRANSFORM_INSUFFICIENT_DATA = 103,
+    TF_TRANSFORM_RESOURCE_LIMIT = 104,
+    TF_TRANSFORM_UNSUPPORTED_VERSION = 105,
+    TF_TRANSFORM_CORRUPT_PLAN = 106,
+    TF_TRANSFORM_NUMERIC_DOMAIN = 107,
+    TF_TRANSFORM_UNKNOWN_CATEGORY = 108,
+    TF_TRANSFORM_CANCELLED = 109,
+    TF_TRANSFORM_ALLOCATION = 110,
+    TF_TRANSFORM_IO = 111,
+    TF_TRANSFORM_INVALID_STATE = 112,
+    TF_TRANSFORM_UNSUPPORTED_RUNTIME = 113,
+    TF_TRANSFORM_INTERNAL = 199
+} tf_transform_code;
+
+enum {
+    TF_VIEW_FLOAT32 = 0x0101,
+    TF_VIEW_FLOAT64 = 0x0102
+};
+
+#define TF_TRANSFORM_SCHEMA_INPUT  1u
+#define TF_TRANSFORM_SCHEMA_OUTPUT 2u
+
+#define TF_TRANSFORM_CANCEL_ROWS_V1       4096u
+#define TF_TRANSFORM_CANCEL_ELEMENTS_V1  65536u
+#define TF_TRANSFORM_CANCEL_BYTES_V1     65536u
+#define TF_TRANSFORM_CANCEL_ITERS_V1      4096u
+#define TF_TRANSFORM_CANCEL_IO_BYTES_V1 1048576u
+
+typedef struct tf_transform_recipe tf_transform_recipe;
+typedef struct tf_transform_analyzer tf_transform_analyzer;
+typedef struct tf_transform_plan tf_transform_plan;
+typedef struct tf_transform_apply tf_transform_apply;
+typedef struct tf_transform_schema tf_transform_schema;
+typedef struct tf_transform_error tf_transform_error;
+
+typedef int (*tf_transform_cancel_fn)(void *user);
+
+typedef struct tf_transform_limits_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    uint64_t max_recipe_bytes;
+    uint64_t max_plan_bytes;
+    uint64_t max_json_depth;
+    uint64_t max_object_keys;
+    uint64_t max_steps;
+    uint64_t max_input_columns;
+    uint64_t max_output_columns;
+    uint64_t max_categories_per_column;
+    uint64_t max_total_categories;
+    uint64_t max_string_bytes;
+    uint64_t max_decoded_string_bytes;
+    uint64_t max_analyzer_rows;
+    uint64_t max_analyzer_input_bytes;
+    uint64_t max_resident_state_bytes;
+    uint64_t max_spill_bytes;
+    uint64_t max_apply_rows;
+    uint64_t max_apply_input_bytes;
+    uint64_t max_output_elements_per_call;
+    uint64_t max_allocation_bytes;
+    uint64_t max_allocations_per_session;
+    uint64_t max_live_handles;
+    uint64_t max_retired_handle_slots;
+} tf_transform_limits_v1;
+
+typedef struct tf_transform_runtime_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    const tf_transform_limits_v1 *limits;
+    const tf_host_policy *host_policy;
+    const uint8_t *spill_dir_utf8;
+    size_t spill_dir_bytes;
+    tf_transform_cancel_fn cancel;
+    void *cancel_user;
+    uint32_t flags;
+    uint32_t reserved;
+} tf_transform_runtime_v1;
+
+typedef struct tf_field_view_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    uint32_t dtype;
+    uint32_t flags;
+    const uint8_t *id_utf8;
+    size_t id_bytes;
+    const uint8_t *name_utf8;
+    size_t name_bytes;
+} tf_field_view_v1;
+
+typedef struct tf_schema_view_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    size_t column_count;
+    const tf_field_view_v1 *fields;
+    size_t fields_bytes;
+} tf_schema_view_v1;
+
+typedef struct tf_column_view_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    const void *data;
+    size_t data_bytes;
+    size_t stride_bytes;
+    const uint8_t *validity;
+    size_t validity_bytes;
+    size_t validity_bit_offset;
+    size_t validity_bit_stride;
+} tf_column_view_v1;
+
+typedef struct tf_table_view_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    size_t row_count;
+    size_t column_count;
+    const tf_column_view_v1 *columns;
+    size_t columns_bytes;
+} tf_table_view_v1;
+
+typedef struct tf_owned_dense_v1 {
+    uint32_t abi_version;
+    uint32_t struct_size;
+    uint32_t dtype;
+    uint32_t flags;
+    size_t rows;
+    size_t columns;
+    void *data;
+    size_t data_bytes;
+} tf_owned_dense_v1;
+
+tf_transform_code tf_transform_limits_init_safe_v1(
+    tf_transform_limits_v1 *out, size_t out_size);
+
+tf_transform_code tf_transform_recipe_from_json(
+    const uint8_t *json, size_t json_len,
+    const tf_transform_limits_v1 *limits,
+    tf_transform_recipe **out, tf_transform_error **error);
+tf_transform_code tf_transform_analyzer_create(
+    const tf_transform_recipe *recipe, const tf_schema_view_v1 *input_schema,
+    const tf_transform_runtime_v1 *runtime,
+    tf_transform_analyzer **out, tf_transform_error **error);
+tf_transform_code tf_transform_analyzer_push(
+    tf_transform_analyzer *analyzer, const tf_table_view_v1 *table,
+    tf_transform_error **error);
+tf_transform_code tf_transform_analyzer_finalize(
+    tf_transform_analyzer *analyzer, tf_transform_plan **out,
+    tf_transform_error **error);
+tf_transform_code tf_transform_plan_export(
+    const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
+    uint8_t **out, size_t *out_len, tf_transform_error **error);
+tf_transform_code tf_transform_plan_import(
+    const uint8_t *bytes, size_t len, const tf_transform_runtime_v1 *runtime,
+    tf_transform_plan **out, tf_transform_error **error);
+tf_transform_code tf_transform_apply_create(
+    const tf_transform_plan *plan, const tf_schema_view_v1 *runtime_schema,
+    const tf_transform_runtime_v1 *runtime,
+    tf_transform_apply **out, tf_transform_error **error);
+tf_transform_code tf_transform_apply_run(
+    tf_transform_apply *apply, const tf_table_view_v1 *table,
+    tf_owned_dense_v1 *out, tf_transform_error **error);
+
+void tf_transform_recipe_destroy(tf_transform_recipe **recipe);
+void tf_transform_analyzer_destroy(tf_transform_analyzer **analyzer);
+void tf_transform_plan_destroy(tf_transform_plan **plan);
+void tf_transform_apply_destroy(tf_transform_apply **apply);
+void tf_transform_error_destroy(tf_transform_error **error);
+void tf_transform_bytes_free(uint8_t **bytes, size_t *len);
+void tf_owned_dense_free(tf_owned_dense_v1 *dense);
+
+tf_transform_code tf_transform_error_get_code(const tf_transform_error *error);
+const uint8_t *tf_transform_error_message(
+    const tf_transform_error *error, size_t *len);
+
+tf_transform_code tf_transform_plan_schema(
+    const tf_transform_plan *plan, uint32_t which,
+    const tf_transform_schema **out, tf_transform_error **error);
+size_t tf_transform_schema_field_count(const tf_transform_schema *schema);
+tf_transform_code tf_transform_schema_field(
+    const tf_transform_schema *schema, size_t index,
+    tf_field_view_v1 *out, tf_transform_error **error);
+tf_transform_code tf_transform_plan_schema_json(
+    const tf_transform_plan *plan, uint32_t which,
+    const tf_transform_limits_v1 *limits,
+    uint8_t **out, size_t *out_len, tf_transform_error **error);
+tf_transform_code tf_transform_plan_recipe_sha256(
+    const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
+    uint8_t out_hex[65], tf_transform_error **error);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif