tranfi 0.1.2 → 0.2.0

package/csrc/internal.h

@@ -12,6 +12,71 @@
 #include <stdint.h>
 #include <stdbool.h>
 
+typedef struct cJSON cJSON;
+
+#define TF_MAX_BATCH_ROWS             ((size_t)64u * 1024u)
+#define TF_MAX_OUTPUT_ROWS_PER_BATCH  ((size_t)1024u * 1024u)
+#define TF_MAX_EXPANDING_ROWS_PER_INPUT TF_MAX_OUTPUT_ROWS_PER_BATCH
+#define TF_MAX_EXPLODE_TOKENS_PER_ROW TF_MAX_OUTPUT_ROWS_PER_BATCH
+#define TF_MAX_COLUMNS                ((size_t)64u * 1024u)
+#define TF_MAX_RECORD_BYTES           ((size_t)1024u * 1024u * 1024u)
+#define TF_MAX_CELL_BYTES             ((size_t)64u * 1024u * 1024u)
+#define TF_MAX_COLUMN_NAME_BYTES      ((size_t)4096u)
+#define TF_MAX_SCHEMA_BYTES           ((size_t)16u * 1024u * 1024u)
+#define TF_MAX_ERROR_BYTES            ((size_t)64u * 1024u * 1024u)
+#define TF_MAX_REGEX_PATTERN_BYTES    ((size_t)4096u)
+#define TF_MAX_REGEX_CELL_BYTES       ((size_t)64u * 1024u)
+#if SIZE_MAX > 9007199254740991ULL
+#define TF_MAX_SAFE_SIZE_ARG          ((size_t)9007199254740991ULL)
+#else
+#define TF_MAX_SAFE_SIZE_ARG          ((size_t)SIZE_MAX)
+#endif
+#define TF_MAX_COUNT_ARG              TF_MAX_SAFE_SIZE_ARG
+#define TF_MAX_STATE_BYTES            TF_MAX_SAFE_SIZE_ARG
+#define TF_MAX_SPILL_MEMORY_BYTES     TF_MAX_SAFE_SIZE_ARG
+#define TF_MAX_SPILL_RUN_ROWS         TF_MAX_COUNT_ARG
+#define TF_MAX_SPILL_OUTPUT_ROWS      TF_MAX_BATCH_ROWS
+#define TF_MAX_AUDIT_RECORDS          ((size_t)1000u * 1000u)
+#define TF_MAX_TABLE_WIDTH            ((size_t)4096u)
+#define TF_MAX_TABLE_ROWS             TF_MAX_COUNT_ARG
+#define TF_MAX_WINDOW_SIZE            TF_MAX_OUTPUT_ROWS_PER_BATCH
+#define TF_FLOAT64_ROUNDTRIP_FORMAT   "%.17g"
+
+#if defined(__GNUC__) || defined(__clang__)
+#define TF_WARN_UNUSED __attribute__((warn_unused_result))
+#else
+#define TF_WARN_UNUSED
+#endif
+
+TF_WARN_UNUSED int tf_format_float64(char *buf, size_t buf_size, double value);
+TF_WARN_UNUSED int tf_size_add(size_t a, size_t b, size_t *out);
+TF_WARN_UNUSED int tf_size_mul(size_t a, size_t b, size_t *out);
+TF_WARN_UNUSED int tf_size_align(size_t x, size_t align, size_t *out);
+TF_WARN_UNUSED int tf_size_grow_pow2(size_t current, size_t min_value, size_t min_capacity, size_t *out);
+TF_WARN_UNUSED void *tf_mallocarray_checked(size_t count, size_t elem_size);
+TF_WARN_UNUSED void *tf_callocarray_checked(size_t count, size_t elem_size);
+TF_WARN_UNUSED void *tf_reallocarray_checked(void *ptr, size_t count, size_t elem_size);
+TF_WARN_UNUSED char *tf_strdup_checked(const char *s);
+TF_WARN_UNUSED char *tf_string_append_suffix_checked(const char *prefix, const char *suffix);
+TF_WARN_UNUSED int tf_check_byte_limit(size_t n, size_t max_value,
+                                       const char *context, const char *name);
+TF_WARN_UNUSED int tf_string_length_bounded(const char *s, size_t max_value,
+                                            size_t *out, const char *context,
+                                            const char *name);
+TF_WARN_UNUSED int tf_json_get_size_arg(const cJSON *args, const char *name,
+                                        size_t min_value, size_t max_value,
+                                        size_t *out, const char *context);
+TF_WARN_UNUSED int tf_json_size_value(const cJSON *item, const char *name,
+                                      size_t min_value, size_t max_value,
+                                      size_t *out, const char *context);
+TF_WARN_UNUSED int tf_json_get_size_arg_any(const cJSON *args, const char *name,
+                                            const char *alt_name,
+                                            size_t min_value, size_t max_value,
+                                            size_t *out, const char *context);
+
+typedef int (*tf_index_compare_fn)(const void *ctx, size_t a, size_t b);
+void tf_sort_indices(size_t *indices, size_t n, tf_index_compare_fn compare, const void *ctx);
+
 /* ---- Arena allocator ---- */
 
 typedef struct tf_arena_block {
@@ -42,13 +107,21 @@ typedef struct tf_buffer {
     size_t   read_pos; /* consumer read position */
 } tf_buffer;
 
-void   tf_buffer_init(tf_buffer *b);
-int    tf_buffer_write(tf_buffer *b, const uint8_t *data, size_t len);
+void tf_buffer_init(tf_buffer *b);
+TF_WARN_UNUSED int tf_buffer_write(tf_buffer *b, const uint8_t *data, size_t len);
 size_t tf_buffer_read(tf_buffer *b, uint8_t *out, size_t len);
 size_t tf_buffer_readable(const tf_buffer *b);
-void   tf_buffer_compact(tf_buffer *b);
-int    tf_buffer_write_str(tf_buffer *b, const char *s);
-void   tf_buffer_free(tf_buffer *b);
+void tf_buffer_compact(tf_buffer *b);
+TF_WARN_UNUSED int tf_buffer_write_str(tf_buffer *b, const char *s);
+TF_WARN_UNUSED int tf_buffer_write_line(tf_buffer *b, const char *s);
+TF_WARN_UNUSED int tf_buffer_write_json_line(tf_buffer *b, const cJSON *obj);
+void tf_buffer_free(tf_buffer *b);
+TF_WARN_UNUSED int tf_json_add_string(cJSON *obj, const char *name, const char *value);
+TF_WARN_UNUSED int tf_json_add_number(cJSON *obj, const char *name, double value);
+TF_WARN_UNUSED int tf_json_add_bool(cJSON *obj, const char *name, int value);
+TF_WARN_UNUSED int tf_json_add_null(cJSON *obj, const char *name);
+TF_WARN_UNUSED int tf_json_add_item(cJSON *obj, const char *name, cJSON *item);
+TF_WARN_UNUSED int tf_json_add_array_item(cJSON *arr, cJSON *item);
 
 /* ---- Columnar batch ---- */
 /* (tf_type is defined in ir.h, included above) */
@@ -62,20 +135,47 @@ typedef struct tf_batch {
     void       **columns;    /* array of typed column arrays */
     uint8_t    **nulls;      /* null bitmap per column (1 byte per row for simplicity) */
     tf_arena    *arena;      /* owns all memory for this batch */
+    size_t       schema_name_bytes;
 } tf_batch;
 
+typedef union tf_cell_value {
+    bool b;
+    int64_t i64;
+    double f64;
+    const char *str;
+    int32_t date;
+} tf_cell_value;
+
+/* Owned string variant used by spill rows before copying into a batch arena. */
+typedef union tf_owned_cell_value {
+    uint8_t b;
+    int64_t i64;
+    double f64;
+    int32_t date;
+    char *str;
+} tf_owned_cell_value;
+
 tf_batch *tf_batch_create(size_t n_cols, size_t capacity);
-int       tf_batch_set_schema(tf_batch *b, size_t col, const char *name, tf_type type);
-int       tf_batch_ensure_capacity(tf_batch *b, size_t min_rows);
+TF_WARN_UNUSED int tf_batch_set_schema(tf_batch *b, size_t col, const char *name, tf_type type);
+TF_WARN_UNUSED int tf_batch_ensure_capacity(tf_batch *b, size_t min_rows);
+TF_WARN_UNUSED int tf_batch_expose_row(tf_batch *b, size_t row);
 
 /* Set a value in a specific cell. String values are copied into the arena. */
-void tf_batch_set_null(tf_batch *b, size_t row, size_t col);
-void tf_batch_set_bool(tf_batch *b, size_t row, size_t col, bool val);
-void tf_batch_set_int64(tf_batch *b, size_t row, size_t col, int64_t val);
-void tf_batch_set_float64(tf_batch *b, size_t row, size_t col, double val);
-void tf_batch_set_string(tf_batch *b, size_t row, size_t col, const char *val);
-void tf_batch_set_date(tf_batch *b, size_t row, size_t col, int32_t val);
-void tf_batch_set_timestamp(tf_batch *b, size_t row, size_t col, int64_t val);
+TF_WARN_UNUSED int tf_batch_set_null(tf_batch *b, size_t row, size_t col);
+TF_WARN_UNUSED int tf_batch_set_bool(tf_batch *b, size_t row, size_t col, bool val);
+TF_WARN_UNUSED int tf_batch_set_int64(tf_batch *b, size_t row, size_t col, int64_t val);
+TF_WARN_UNUSED int tf_batch_set_float64(tf_batch *b, size_t row, size_t col, double val);
+TF_WARN_UNUSED int tf_batch_set_string(tf_batch *b, size_t row, size_t col, const char *val);
+TF_WARN_UNUSED int tf_batch_set_string_len(tf_batch *b, size_t row, size_t col,
+                                           const char *val, size_t len);
+TF_WARN_UNUSED int tf_batch_set_date(tf_batch *b, size_t row, size_t col, int32_t val);
+TF_WARN_UNUSED int tf_batch_set_timestamp(tf_batch *b, size_t row, size_t col, int64_t val);
+TF_WARN_UNUSED int tf_batch_set_cell_value(tf_batch *b, size_t row, size_t col,
+                                           tf_type type, int is_null,
+                                           const tf_cell_value *value);
+TF_WARN_UNUSED int tf_batch_set_owned_cell_value(tf_batch *b, size_t row, size_t col,
+                                                 tf_type type, int is_null,
+                                                 const tf_owned_cell_value *value);
 
 /* Get values from a cell. */
 bool      tf_batch_is_null(const tf_batch *b, size_t row, size_t col);
@@ -89,11 +189,113 @@ int64_t   tf_batch_get_timestamp(const tf_batch *b, size_t row, size_t col);
 /* Find column index by name. Returns -1 if not found. */
 int tf_batch_col_index(const tf_batch *b, const char *name);
 
-/* Copy a single row from src to dst batch. */
-int tf_batch_copy_row(tf_batch *dst, size_t dst_row,
-                      const tf_batch *src, size_t src_row);
+/* Schema and row-copy helpers. */
+typedef enum tf_cell_string_format {
+    TF_CELL_STRING_ROUNDTRIP,   /* round-trip floats, ISO dates/timestamps */
+    TF_CELL_STRING_HUMAN,       /* short floats, ISO dates/timestamps */
+    TF_CELL_STRING_NUMERIC_TIME /* round-trip floats, numeric dates/timestamps */
+} tf_cell_string_format;
+
+TF_WARN_UNUSED int tf_batch_clone_schema(tf_batch *dst, const tf_batch *src);
+TF_WARN_UNUSED int tf_batch_clone_with_extra_cols(tf_batch *dst, const tf_batch *src,
+                                                  const char **names, const tf_type *types,
+                                                  size_t n_extra);
+TF_WARN_UNUSED int tf_batch_copy_cell(tf_batch *dst, size_t dst_row, size_t dst_col,
+                                      const tf_batch *src, size_t src_row, size_t src_col);
+TF_WARN_UNUSED int tf_batch_copy_cell_index(tf_batch *dst, size_t dst_row, size_t dst_col,
+                                            const tf_batch *src, size_t src_row, int src_col);
+TF_WARN_UNUSED int tf_batch_format_cell_as_string(const tf_batch *src, size_t src_row, size_t src_col,
+                                                  tf_cell_string_format format,
+                                                  char *buf, size_t buf_size,
+                                                  const char **out);
+TF_WARN_UNUSED int tf_batch_copy_cell_as_string(tf_batch *dst, size_t dst_row, size_t dst_col,
+                                                const tf_batch *src, size_t src_row, size_t src_col);
+TF_WARN_UNUSED int tf_batch_copy_selected_row(tf_batch *dst, size_t dst_row,
+                                              const tf_batch *src, size_t src_row,
+                                              const size_t *cols, size_t n_cols);
+TF_WARN_UNUSED int tf_batch_copy_selected_columns(tf_batch *dst,
+                                                  const tf_batch *src,
+                                                  const size_t *cols,
+                                                  size_t n_cols);
+TF_WARN_UNUSED int tf_batch_copy_row(tf_batch *dst, size_t dst_row,
+                                     const tf_batch *src, size_t src_row);
+TF_WARN_UNUSED int tf_batch_clone_with_selected_extra_cols(tf_batch *dst,
+                                                           const tf_batch *src,
+                                                           const tf_batch *extra,
+                                                           const int *extra_cols,
+                                                           size_t n_extra);
+TF_WARN_UNUSED int tf_batch_append_row(tf_batch *dst, const tf_batch *src,
+                                       size_t src_row);
+TF_WARN_UNUSED int tf_batch_append_row_with_selected_extra(tf_batch *dst,
+                                                           const tf_batch *left,
+                                                           size_t left_row,
+                                                           const tf_batch *extra,
+                                                           size_t extra_row,
+                                                           const int *extra_cols,
+                                                           size_t n_extra);
+TF_WARN_UNUSED int tf_batch_append_row_with_null_extra(tf_batch *dst,
+                                                       const tf_batch *left,
+                                                       size_t left_row,
+                                                       size_t n_extra);
 
 void tf_batch_free(tf_batch *b);
+void tf_batch_array_free_items(tf_batch **batches, size_t n_batches);
+void tf_batch_array_free(tf_batch **batches, size_t n_batches);
+
+/* ---- Audit / side-channel row serialization helpers ---- */
+
+typedef struct tf_audit_options {
+    int include_row;
+    char **columns;
+    size_t n_columns;
+    char **redact_columns;
+    size_t n_redact_columns;
+    char **hash_columns;
+    size_t n_hash_columns;
+    size_t max_bytes;
+    size_t max_cell_bytes;
+} tf_audit_options;
+
+void tf_audit_options_init(tf_audit_options *opts, int default_include_row);
+int  tf_audit_options_parse(tf_audit_options *opts, const cJSON *args, const char *context);
+void tf_audit_options_free(tf_audit_options *opts);
+int  tf_audit_column_is_redacted(const tf_audit_options *opts, const char *column);
+int  tf_audit_column_is_hashed(const tf_audit_options *opts, const char *column);
+int  tf_audit_hash_string(const char *value, char *out, size_t out_size);
+const char *tf_audit_format_string_for_column(const tf_audit_options *opts, const char *column,
+                                              const char *value, char *buf, size_t buf_size);
+TF_WARN_UNUSED char *tf_audit_format_string_dup_for_column(const tf_audit_options *opts,
+                                                           const char *column,
+                                                           const char *value);
+cJSON *tf_audit_string_to_json_for_column(const tf_audit_options *opts, const char *column,
+                                          const char *value);
+TF_WARN_UNUSED int tf_json_add_audit_string(cJSON *obj, const char *name,
+                                            const tf_audit_options *opts,
+                                            const char *column, const char *value);
+cJSON *tf_audit_cell_to_json(const tf_batch *b, size_t row, size_t col, const tf_audit_options *opts);
+cJSON *tf_audit_row_to_json(const tf_batch *b, size_t row, const tf_audit_options *opts);
+
+/* ---- Column selector helpers ---- */
+
+int tf_json_path_validate(const char *path);
+const cJSON *tf_json_path_resolve(const cJSON *root, const char *path);
+
+int tf_column_selector_has_syntax(const char *raw);
+int tf_column_selectors_have_syntax_json(const cJSON *selectors);
+int tf_column_selectors_resolve(char **selectors, size_t n_selectors,
+                                char **names, const tf_type *types, size_t n_cols,
+                                int **out_indices, size_t *out_n,
+                                char **error);
+int tf_column_selectors_resolve_json(const cJSON *selectors,
+                                     char **names, const tf_type *types, size_t n_cols,
+                                     int **out_indices, size_t *out_n,
+                                     char **error);
+
+int tf_estimate_step_state_bytes(const tf_ir_node *node, size_t *out,
+                                 char *reason, size_t reason_size);
+int tf_estimate_key_state_plan_bytes(const tf_ir_plan *ir, size_t *out,
+                                     const tf_ir_node **failed_node,
+                                     char *reason, size_t reason_size);
 
 /* ---- Step interface (transforms) ---- */
 
@@ -105,8 +307,18 @@ typedef struct tf_side_channels {
     tf_buffer *errors;
     tf_buffer *stats;
     tf_buffer *samples;
+    const char *source_name;
 } tf_side_channels;
 
+TF_WARN_UNUSED int tf_side_write_error(tf_side_channels *side, const char *msg);
+
+/* Hidden IR args added by the built-in workspace_root resolver. */
+#define TF_POLICY_VALIDATED_FILE_PATH_ARG       "_tf_validated_file_path"
+#define TF_POLICY_VALIDATED_RULES_FILE_PATH_ARG "_tf_validated_rules_file_path"
+
+const char *tf_policy_validated_path_arg(const cJSON *args, const char *path_arg);
+FILE *tf_policy_fopen_read(const char *path, const char *validated_path);
+
 typedef struct tf_step {
     /* Process one input batch, produce zero or one output batch.
      * *out is set to a new batch (caller frees) or NULL if filtered away. */
@@ -114,11 +326,38 @@ typedef struct tf_step {
                     tf_side_channels *side);
     /* Flush any buffered state. */
     int  (*flush)(struct tf_step *self, tf_batch **out, tf_side_channels *side);
+    /* Optional continuation for flush-latent steps that emit multiple batches. */
+    int  (*flush_next)(struct tf_step *self, tf_batch **out, tf_side_channels *side);
+    /* Optional JSON stats hook. Appends comma-prefixed fields to step_stats. */
+    int  (*append_stats)(struct tf_step *self, tf_buffer *out);
     /* Free all resources. */
     void (*destroy)(struct tf_step *self);
     void  *state;
 } tf_step;
 
+#define TF_STEP_WARN_BLOCKING              (1u << 0)
+#define TF_STEP_WARN_FLUSH_LATENT          (1u << 1)
+#define TF_STEP_WARN_UNBOUNDED_STATE       (1u << 2)
+#define TF_STEP_WARN_DATA_DEPENDENT_SCHEMA (1u << 3)
+
+typedef struct tf_step_run_stats {
+    char           *op;
+    char           *state_estimate;
+    char           *state_bytes_reason;
+    char           *execution_target;
+    size_t          node_index;
+    size_t          state_bytes_estimate;
+    int             has_state_bytes_estimate;
+    uint32_t        warnings;
+    tf_memory_class memory_class;
+    tf_emit_class   emit_class;
+    tf_schema_class schema_class;
+    size_t          batches_in;
+    size_t          batches_out;
+    size_t          rows_in;
+    size_t          rows_out;
+} tf_step_run_stats;
+
 /* ---- Decoder interface (bytes → batches) ---- */
 
 typedef struct tf_decoder {
@@ -128,9 +367,10 @@ typedef struct tf_decoder {
      * Caller frees the array and each batch.
      */
     int  (*decode)(struct tf_decoder *self, const uint8_t *data, size_t len,
-                   tf_batch ***out, size_t *n_out);
+                   tf_batch ***out, size_t *n_out, tf_side_channels *side);
     /* Flush remaining data (e.g. last partial line). */
-    int  (*flush)(struct tf_decoder *self, tf_batch ***out, size_t *n_out);
+    int  (*flush)(struct tf_decoder *self, tf_batch ***out, size_t *n_out,
+                  tf_side_channels *side);
     void (*destroy)(struct tf_decoder *self);
     void  *state;
 } tf_decoder;
@@ -154,21 +394,41 @@ struct tf_pipeline {
     tf_decoder  *decoder;
     tf_step    **steps;
     size_t       n_steps;
+    tf_step_run_stats *step_stats;
+    size_t       n_step_stats;
     tf_encoder  *encoder;
     tf_buffer    output[TF_NUM_CHANNELS];
+    tf_pipeline_sink_fn sinks[TF_NUM_CHANNELS];
+    void        *sink_users[TF_NUM_CHANNELS];
+    tf_pipeline_batch_sink_fn batch_sink;
+    void        *batch_sink_user;
+    int          encode_output;
+    tf_pipeline_progress_fn progress_cb;
+    void        *progress_user;
+    size_t       progress_interval_rows;
+    size_t       progress_last_rows_in;
+    size_t       progress_last_rows_out;
     tf_side_channels side;
     size_t       rows_in;
     size_t       rows_out;
+    size_t       batches_in;
+    size_t       batches_out;
     size_t       bytes_in;
     size_t       bytes_out;
+    char        *source_name;
     char        *error;
     int          finished;
+    int          finish_started;
+    int          finish_phase;
+    tf_batch   **finish_batches;
+    size_t       finish_n_batches;
+    size_t       finish_batch_index;
+    size_t       finish_step_index;
+    int          finish_step_in_next;
 };
 
 /* ---- Codec constructors (used by plan parser) ---- */
 
-typedef struct cJSON cJSON;
-
 tf_decoder *tf_csv_decoder_create(const cJSON *args);
 tf_encoder *tf_csv_encoder_create(const cJSON *args);
 tf_decoder *tf_jsonl_decoder_create(const cJSON *args);
@@ -180,14 +440,23 @@ tf_encoder *tf_text_encoder_create(const cJSON *args);
 
 tf_step *tf_filter_create(const cJSON *args);
 tf_step *tf_select_create(const cJSON *args);
+tf_step *tf_relocate_create(const cJSON *args);
 tf_step *tf_rename_create(const cJSON *args);
 tf_step *tf_head_create(const cJSON *args);
 tf_step *tf_skip_create(const cJSON *args);
 tf_step *tf_derive_create(const cJSON *args);
+tf_step *tf_source_name_create(const cJSON *args);
+tf_step *tf_across_create(const cJSON *args);
 tf_step *tf_stats_create(const cJSON *args);
+tf_step *tf_scan_create(const cJSON *args);
 tf_step *tf_unique_create(const cJSON *args);
 tf_step *tf_sort_create(const cJSON *args);
 tf_step *tf_validate_create(const cJSON *args);
+tf_step *tf_assert_create(const cJSON *args);
+tf_step *tf_quarantine_create(const cJSON *args);
+tf_step *tf_schema_create(const cJSON *args);
+tf_step *tf_schema_infer_create(const cJSON *args);
+tf_step *tf_tee_create(const cJSON *args);
 tf_step *tf_trim_create(const cJSON *args);
 tf_step *tf_fill_null_create(const cJSON *args);
 tf_step *tf_cast_create(const cJSON *args);
@@ -198,20 +467,46 @@ tf_step *tf_bin_create(const cJSON *args);
 tf_step *tf_fill_down_create(const cJSON *args);
 tf_step *tf_step_create(const cJSON *args);
 tf_step *tf_window_create(const cJSON *args);
+tf_step *tf_rolling_sum_create(const cJSON *args);
+tf_step *tf_rolling_mean_create(const cJSON *args);
+tf_step *tf_rolling_min_create(const cJSON *args);
+tf_step *tf_rolling_max_create(const cJSON *args);
+tf_step *tf_rolling_any_create(const cJSON *args);
+tf_step *tf_rolling_all_create(const cJSON *args);
 tf_step *tf_explode_create(const cJSON *args);
 tf_step *tf_split_create(const cJSON *args);
 tf_step *tf_unpivot_create(const cJSON *args);
 tf_step *tf_tail_create(const cJSON *args);
 tf_step *tf_top_create(const cJSON *args);
+tf_step *tf_top_k_create(const cJSON *args);
+tf_step *tf_bottom_k_create(const cJSON *args);
+tf_step *tf_slice_min_create(const cJSON *args);
+tf_step *tf_slice_max_create(const cJSON *args);
 tf_step *tf_sample_create(const cJSON *args);
 tf_step *tf_group_agg_create(const cJSON *args);
 tf_step *tf_frequency_create(const cJSON *args);
 tf_step *tf_datetime_create(const cJSON *args);
 tf_step *tf_grep_create(const cJSON *args);
+tf_step *tf_json_extract_create(const cJSON *args);
+tf_step *tf_json_filter_create(const cJSON *args);
+tf_step *tf_json_schema_create(const cJSON *args);
+tf_step *tf_json_flatten_create(const cJSON *args);
 tf_step *tf_pivot_create(const cJSON *args);
 tf_step *tf_join_create(const cJSON *args);
+tf_step *tf_semi_join_create(const cJSON *args);
+tf_step *tf_anti_join_create(const cJSON *args);
+tf_step *tf_intersect_create(const cJSON *args);
+tf_step *tf_setdiff_create(const cJSON *args);
+tf_step *tf_intersect_all_create(const cJSON *args);
+tf_step *tf_setdiff_all_create(const cJSON *args);
+tf_step *tf_union_create(const cJSON *args);
+tf_step *tf_union_all_create(const cJSON *args);
 tf_step *tf_stack_create(const cJSON *args);
 tf_step *tf_lead_create(const cJSON *args);
+tf_step *tf_lag_create(const cJSON *args);
+tf_step *tf_shift_create(const cJSON *args);
+tf_step *tf_rowid_create(const cJSON *args);
+tf_step *tf_rleid_create(const cJSON *args);
 tf_step *tf_date_trunc_create(const cJSON *args);
 tf_step *tf_onehot_create(const cJSON *args);
 tf_step *tf_label_encode_create(const cJSON *args);
@@ -237,12 +532,6 @@ int         tf_expr_eval_val(const tf_expr *e, const tf_batch *batch, size_t row
                              tf_eval_result *result);
 void        tf_expr_free(tf_expr *e);
 
-/* ---- Plan parser ---- */
-
-int tf_plan_parse(const char *json, size_t len,
-                  tf_decoder **decoder, tf_step ***steps, size_t *n_steps,
-                  tf_encoder **encoder, char **error);
-
 /* ---- Global error ---- */
 
 void tf_set_last_error(const char *msg);

package/csrc/ir.c

@@ -2,7 +2,7 @@
  * ir.c — IR plan construction, cloning, and memory management.
  */
 
-#include "ir.h"
+#include "internal.h"
 #include "cJSON.h"
 #include <stdlib.h>
 #include <string.h>
@@ -11,8 +11,10 @@
 
 void tf_schema_free(tf_schema *s) {
     if (!s) return;
-    for (size_t i = 0; i < s->n_cols; i++) {
-        free(s->col_names[i]);
+    if (s->col_names) {
+        for (size_t i = 0; i < s->n_cols; i++) {
+            free(s->col_names[i]);
+        }
     }
     free(s->col_names);
     free(s->col_types);
@@ -22,20 +24,38 @@ void tf_schema_free(tf_schema *s) {
     s->known = false;
 }
 
-void tf_schema_copy(tf_schema *dst, const tf_schema *src) {
+int tf_schema_copy_checked(tf_schema *dst, const tf_schema *src) {
+    if (!dst) return TF_ERROR;
+    dst->known = false;
+    dst->n_cols = 0;
+    dst->col_names = NULL;
+    dst->col_types = NULL;
+    if (!src) return TF_OK;
+
     dst->known = src->known;
     dst->n_cols = src->n_cols;
-    if (src->n_cols == 0 || !src->known) {
-        dst->col_names = NULL;
-        dst->col_types = NULL;
-        return;
-    }
+    if (src->n_cols == 0 || !src->known) return TF_OK;
+
     dst->col_names = calloc(src->n_cols, sizeof(char *));
     dst->col_types = calloc(src->n_cols, sizeof(tf_type));
+    if (!dst->col_names || !dst->col_types) {
+        tf_schema_free(dst);
+        return TF_ERROR;
+    }
     for (size_t i = 0; i < src->n_cols; i++) {
-        dst->col_names[i] = strdup(src->col_names[i]);
+        dst->col_names[i] = strdup(src->col_names[i] ? src->col_names[i] : "");
+        if (!dst->col_names[i]) {
+            tf_schema_free(dst);
+            return TF_ERROR;
+        }
         dst->col_types[i] = src->col_types[i];
     }
+    dst->known = true;
+    return TF_OK;
+}
+
+void tf_schema_copy(tf_schema *dst, const tf_schema *src) {
+    (void)tf_schema_copy_checked(dst, src);
 }
 
 /* ---- IR node helpers ---- */
@@ -59,9 +79,12 @@ tf_ir_plan *tf_ir_plan_create(void) {
 }
 
 int tf_ir_plan_add_node(tf_ir_plan *plan, const char *op, cJSON *args) {
+    if (!plan || !op) return -1;
     if (plan->n_nodes >= plan->capacity) {
-        size_t new_cap = plan->capacity * 2;
-        tf_ir_node *new_nodes = realloc(plan->nodes, new_cap * sizeof(tf_ir_node));
+        size_t new_cap = 0;
+        if (tf_size_mul(plan->capacity, 2, &new_cap) != TF_OK) return -1;
+        tf_ir_node *new_nodes = tf_reallocarray_checked(plan->nodes, new_cap,
+                                                        sizeof(tf_ir_node));
         if (!new_nodes) return -1;
         memset(new_nodes + plan->capacity, 0,
                (new_cap - plan->capacity) * sizeof(tf_ir_node));
@@ -69,10 +92,18 @@ int tf_ir_plan_add_node(tf_ir_plan *plan, const char *op, cJSON *args) {
         plan->capacity = new_cap;
     }
 
+    char *op_copy = strdup(op);
+    cJSON *args_copy = args ? cJSON_Duplicate(args, 1) : cJSON_CreateObject();
+    if (!op_copy || !args_copy) {
+        free(op_copy);
+        if (args_copy) cJSON_Delete(args_copy);
+        return -1;
+    }
+
     tf_ir_node *node = &plan->nodes[plan->n_nodes];
     memset(node, 0, sizeof(tf_ir_node));
-    node->op = strdup(op);
-    node->args = args ? cJSON_Duplicate(args, 1) : cJSON_CreateObject();
+    node->op = op_copy;
+    node->args = args_copy;
     node->index = plan->n_nodes;
 
     plan->n_nodes++;
@@ -93,16 +124,32 @@ tf_ir_plan *tf_ir_plan_clone(const tf_ir_plan *plan) {
         }
         /* Copy schemas if they were inferred */
         tf_ir_node *dst = &clone->nodes[i];
-        tf_schema_copy(&dst->input_schema, &src->input_schema);
-        tf_schema_copy(&dst->output_schema, &src->output_schema);
+        if (tf_schema_copy_checked(&dst->input_schema, &src->input_schema) != TF_OK ||
+            tf_schema_copy_checked(&dst->output_schema, &src->output_schema) != TF_OK) {
+            tf_ir_plan_free(clone);
+            return NULL;
+        }
         dst->caps = src->caps;
+        dst->memory_class = src->memory_class;
+        dst->emit_class = src->emit_class;
+        dst->schema_class = src->schema_class;
+        dst->state_estimate = src->state_estimate;
     }
 
-    tf_schema_copy(&clone->final_schema, &plan->final_schema);
+    if (tf_schema_copy_checked(&clone->final_schema, &plan->final_schema) != TF_OK) {
+        tf_ir_plan_free(clone);
+        return NULL;
+    }
     clone->plan_caps = plan->plan_caps;
     clone->validated = plan->validated;
     clone->schema_inferred = plan->schema_inferred;
-    if (plan->error) clone->error = strdup(plan->error);
+    if (plan->error) {
+        clone->error = strdup(plan->error);
+        if (!clone->error) {
+            tf_ir_plan_free(clone);
+            return NULL;
+        }
+    }
 
     return clone;
 }