tranfi 0.0.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/LICENSE +177 -21
  2. package/NOTICE +8 -0
  3. package/README.md +627 -0
  4. package/app/assets/index-6quYZ5Ap.css +5 -0
  5. package/app/assets/index-BIAIKnrp.js +160 -0
  6. package/app/assets/materialdesignicons-webfont-B7mPwVP_.ttf +0 -0
  7. package/app/assets/materialdesignicons-webfont-CSr8KVlo.eot +0 -0
  8. package/app/assets/materialdesignicons-webfont-Dp5v-WZN.woff2 +0 -0
  9. package/app/assets/materialdesignicons-webfont-PXm3-2wK.woff +0 -0
  10. package/app/index.html +13 -0
  11. package/binding.gyp +121 -0
  12. package/csrc/arena.c +93 -0
  13. package/csrc/batch.c +976 -0
  14. package/csrc/buffer.c +154 -0
  15. package/csrc/cJSON.c +3386 -0
  16. package/csrc/cJSON.h +316 -0
  17. package/csrc/codec_csv.c +1951 -0
  18. package/csrc/codec_jsonl.c +1086 -0
  19. package/csrc/codec_table.c +248 -0
  20. package/csrc/codec_text.c +447 -0
  21. package/csrc/compiler.c +130 -0
  22. package/csrc/config.h +21 -0
  23. package/csrc/date_utils.h +94 -0
  24. package/csrc/dsl.c +5417 -0
  25. package/csrc/dsl.h +22 -0
  26. package/csrc/expr.c +1553 -0
  27. package/csrc/expr.h +58 -0
  28. package/csrc/internal.h +539 -0
  29. package/csrc/ir.c +166 -0
  30. package/csrc/ir.h +208 -0
  31. package/csrc/ir_schema.c +75 -0
  32. package/csrc/ir_serialize.c +166 -0
  33. package/csrc/ir_sql.c +1822 -0
  34. package/csrc/ir_validate.c +576 -0
  35. package/csrc/json_path.c +210 -0
  36. package/csrc/main.c +1241 -0
  37. package/csrc/memory_estimate.c +477 -0
  38. package/csrc/op_acf.c +283 -0
  39. package/csrc/op_across.c +477 -0
  40. package/csrc/op_anomaly.c +255 -0
  41. package/csrc/op_assert.c +761 -0
  42. package/csrc/op_bin.c +248 -0
  43. package/csrc/op_cast.c +523 -0
  44. package/csrc/op_clip.c +99 -0
  45. package/csrc/op_date_trunc.c +355 -0
  46. package/csrc/op_datetime.c +394 -0
  47. package/csrc/op_derive.c +216 -0
  48. package/csrc/op_diff.c +250 -0
  49. package/csrc/op_ewma.c +222 -0
  50. package/csrc/op_explode.c +206 -0
  51. package/csrc/op_fill_down.c +235 -0
  52. package/csrc/op_fill_null.c +268 -0
  53. package/csrc/op_filter.c +181 -0
  54. package/csrc/op_frequency.c +721 -0
  55. package/csrc/op_grep.c +181 -0
  56. package/csrc/op_group_agg.c +1956 -0
  57. package/csrc/op_hash.c +159 -0
  58. package/csrc/op_head.c +84 -0
  59. package/csrc/op_interpolate.c +445 -0
  60. package/csrc/op_join.c +2902 -0
  61. package/csrc/op_json_extract.c +227 -0
  62. package/csrc/op_json_filter.c +384 -0
  63. package/csrc/op_json_flatten.c +293 -0
  64. package/csrc/op_json_schema.c +503 -0
  65. package/csrc/op_label_encode.c +419 -0
  66. package/csrc/op_lag.c +181 -0
  67. package/csrc/op_lead.c +242 -0
  68. package/csrc/op_normalize.c +510 -0
  69. package/csrc/op_onehot.c +457 -0
  70. package/csrc/op_pivot.c +1754 -0
  71. package/csrc/op_quarantine.c +189 -0
  72. package/csrc/op_registry.c +3044 -0
  73. package/csrc/op_rename.c +129 -0
  74. package/csrc/op_replace.c +354 -0
  75. package/csrc/op_rleid.c +297 -0
  76. package/csrc/op_rowid.c +559 -0
  77. package/csrc/op_sample.c +158 -0
  78. package/csrc/op_schema.c +1341 -0
  79. package/csrc/op_schema_infer.c +252 -0
  80. package/csrc/op_select.c +340 -0
  81. package/csrc/op_set.c +3449 -0
  82. package/csrc/op_skip.c +95 -0
  83. package/csrc/op_sort.c +819 -0
  84. package/csrc/op_source_name.c +120 -0
  85. package/csrc/op_split.c +151 -0
  86. package/csrc/op_split_data.c +119 -0
  87. package/csrc/op_stack.c +271 -0
  88. package/csrc/op_stats.c +875 -0
  89. package/csrc/op_step.c +333 -0
  90. package/csrc/op_tail.c +105 -0
  91. package/csrc/op_tee.c +338 -0
  92. package/csrc/op_top.c +357 -0
  93. package/csrc/op_trim.c +138 -0
  94. package/csrc/op_unique.c +1343 -0
  95. package/csrc/op_unpivot.c +193 -0
  96. package/csrc/op_validate.c +648 -0
  97. package/csrc/op_window.c +591 -0
  98. package/csrc/path_policy.c +85 -0
  99. package/csrc/pipeline.c +1088 -0
  100. package/csrc/recipes.c +104 -0
  101. package/csrc/recipes.h +27 -0
  102. package/csrc/report.c +506 -0
  103. package/csrc/report.h +22 -0
  104. package/csrc/selector.c +1097 -0
  105. package/csrc/size_utils.c +348 -0
  106. package/csrc/spill.c +317 -0
  107. package/csrc/spill.h +21 -0
  108. package/csrc/tranfi.h +291 -0
  109. package/csrc/transform.h +209 -0
  110. package/csrc/transform_api.c +2237 -0
  111. package/csrc/transform_categorical.c +923 -0
  112. package/csrc/transform_internal.h +472 -0
  113. package/csrc/transform_json.c +3812 -0
  114. package/csrc/transform_numeric.c +1966 -0
  115. package/csrc/transform_sha256.c +154 -0
  116. package/csrc/transform_wasm.h +162 -0
  117. package/csrc/transform_wasm_api.c +1373 -0
  118. package/csrc/wasm_api.c +218 -0
  119. package/napi_api.c +534 -0
  120. package/napi_transform.c +1648 -0
  121. package/napi_transform.h +8 -0
  122. package/package.json +64 -59
  123. package/scripts/install-native.js +76 -0
  124. package/scripts/prepack.js +64 -0
  125. package/scripts/sync-csrc.js +23 -0
  126. package/src/cli.js +190 -0
  127. package/src/engines/duckdb.js +142 -0
  128. package/src/index.js +925 -0
  129. package/src/memory_policy.js +411 -0
  130. package/src/native.js +18 -0
  131. package/src/pipeline.js +709 -0
  132. package/src/recipe_json.js +80 -0
  133. package/src/server.js +279 -0
  134. package/src/transform.js +403 -0
  135. package/src/transform_error.js +10 -0
  136. package/src/wasm.js +21 -0
  137. package/wasm/index.js +732 -0
  138. package/wasm/package.json +1 -0
  139. package/wasm/tranfi_core.js +0 -0
  140. package/wasm/transform.js +1156 -0
  141. package/wasm/worker.js +786 -0
  142. package/dist/bundle.js +0 -1
  143. package/index.html +0 -18
  144. package/src/app.css +0 -169
  145. package/src/app.js +0 -203
  146. package/src/app.vue +0 -250
  147. package/src/bulma-input.vue +0 -110
  148. package/src/common-inputs.js +0 -28
  149. package/src/main.js +0 -20
  150. package/src/transforms.js +0 -166
  151. package/webpack.config.js +0 -108
@@ -0,0 +1,3812 @@
1
+ #include "transform_internal.h"
2
+
3
+ #include <math.h>
4
+ #include <stdio.h>
5
+ #include <stdlib.h>
6
+ #include <string.h>
7
+
8
+ typedef struct tf_json_work {
9
+ const tf_transform_runtime_copy *runtime;
10
+ tf_transform_resource_ledger *ledger;
11
+ tf_transform_error **error;
12
+ size_t bytes_since_poll;
13
+ size_t iterations_since_poll;
14
+ int cancelled;
15
+ } tf_json_work;
16
+
17
+ typedef struct tf_json_scan_metrics {
18
+ uint64_t node_count_upper;
19
+ uint64_t string_count;
20
+ uint64_t number_count;
21
+ uint64_t object_count;
22
+ uint64_t decoded_string_bytes;
23
+ uint64_t max_string_bytes;
24
+ uint64_t max_number_bytes;
25
+ uint64_t resident_bytes_persistent;
26
+ uint64_t resident_bytes_upper;
27
+ uint64_t allocation_count_upper;
28
+ } tf_json_scan_metrics;
29
+
30
+ static tf_transform_code json_work_progress(
31
+ tf_json_work *work, size_t bytes, size_t iterations, int force) {
32
+ tf_transform_code code;
33
+ if (!work || !work->runtime) return TF_TRANSFORM_OK;
34
+ if (bytes > SIZE_MAX - work->bytes_since_poll
35
+ || iterations > SIZE_MAX - work->iterations_since_poll)
36
+ return tf_transform_set_error(
37
+ work->error, TF_TRANSFORM_RESOURCE_LIMIT,
38
+ "JSON progress counter overflows");
39
+ work->bytes_since_poll += bytes;
40
+ work->iterations_since_poll += iterations;
41
+ if (!force
42
+ && work->bytes_since_poll < TF_TRANSFORM_CANCEL_BYTES_V1
43
+ && work->iterations_since_poll < TF_TRANSFORM_CANCEL_ITERS_V1)
44
+ return TF_TRANSFORM_OK;
45
+ code = tf_transform_poll_cancel(work->runtime, work->error);
46
+ if (code != TF_TRANSFORM_OK) {
47
+ work->cancelled = 1;
48
+ return code;
49
+ }
50
+ work->bytes_since_poll = 0;
51
+ work->iterations_since_poll = 0;
52
+ return TF_TRANSFORM_OK;
53
+ }
54
+
55
+ static cJSON_bool json_parser_poll(void *user_data) {
56
+ tf_json_work *work = (tf_json_work *)user_data;
57
+ if (!work || !work->runtime || !work->runtime->cancel) return 1;
58
+ if (work->runtime->cancel(work->runtime->cancel_user)) {
59
+ work->cancelled = 1;
60
+ return 0;
61
+ }
62
+ return 1;
63
+ }
64
+
65
+ static tf_transform_code json_work_allocation(
66
+ tf_json_work *work, uint64_t bytes, tf_transform_error **error) {
67
+ if (!work || !work->ledger) return tf_transform_set_error(
68
+ error, TF_TRANSFORM_INTERNAL, "JSON resource ledger is missing");
69
+ return tf_transform_resource_reserve(work->ledger, bytes, error);
70
+ }
71
+
72
+ static void json_work_release(tf_json_work *work, uint64_t bytes) {
73
+ if (work) tf_transform_resource_release(work->ledger, bytes);
74
+ }
75
+
76
+ static tf_transform_code buffer_reserve(
77
+ tf_transform_buffer *buffer, size_t extra,
78
+ const tf_transform_limits_v1 *limits, tf_json_work *work,
79
+ tf_transform_error **error) {
80
+ size_t needed;
81
+ size_t capacity;
82
+ uint8_t *data;
83
+ if (extra > SIZE_MAX - buffer->len)
84
+ return tf_transform_set_error(
85
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON size overflow");
86
+ needed = buffer->len + extra;
87
+ if (needed > limits->max_plan_bytes || needed > limits->max_allocation_bytes)
88
+ return tf_transform_set_error(
89
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON exceeds byte limit");
90
+ if (buffer->count_only || needed <= buffer->cap) return TF_TRANSFORM_OK;
91
+ capacity = needed;
92
+ if (buffer->data || capacity != buffer->cap)
93
+ return tf_transform_set_error(
94
+ error, TF_TRANSFORM_INTERNAL,
95
+ "canonical JSON fixed buffer is too small");
96
+ (void)work;
97
+ (void)data;
98
+ return TF_TRANSFORM_OK;
99
+ }
100
+
101
+ static tf_transform_code buffer_write(
102
+ tf_transform_buffer *buffer, const void *data, size_t len,
103
+ const tf_transform_limits_v1 *limits, tf_json_work *work,
104
+ tf_transform_error **error) {
105
+ tf_transform_code code = buffer_reserve(
106
+ buffer, len, limits, work, error);
107
+ if (code != TF_TRANSFORM_OK) return code;
108
+ if (len != 0 && (!buffer->count_only && !buffer->data))
109
+ return tf_transform_set_error(
110
+ error, TF_TRANSFORM_INTERNAL, "canonical JSON buffer invariant failed");
111
+ if (len != 0 && !data)
112
+ return tf_transform_set_error(
113
+ error, TF_TRANSFORM_INTERNAL, "canonical JSON source is null");
114
+ if (len != 0 && !buffer->count_only)
115
+ memcpy(buffer->data + buffer->len, data, len);
116
+ buffer->len += len;
117
+ return json_work_progress(work, len, 0, 0);
118
+ }
119
+
120
+ static tf_transform_code write_json_string(
121
+ tf_transform_buffer *buffer, const char *text,
122
+ const tf_transform_limits_v1 *limits, tf_json_work *work,
123
+ tf_transform_error **error) {
124
+ static const char hex[] = "0123456789abcdef";
125
+ tf_transform_code code;
126
+ size_t len = 0;
127
+ if (!text) return tf_transform_set_error(
128
+ error, TF_TRANSFORM_INTERNAL, "JSON string is null");
129
+ while (len <= limits->max_string_bytes && text[len] != '\0') {
130
+ code = json_work_progress(work, 1, 0, 0);
131
+ if (code != TF_TRANSFORM_OK) return code;
132
+ ++len;
133
+ }
134
+ if (len > limits->max_string_bytes) return tf_transform_set_error(
135
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON string exceeds limit");
136
+ code = buffer_write(buffer, "\"", 1, limits, work, error);
137
+ if (code != TF_TRANSFORM_OK) return code;
138
+ for (size_t i = 0; i < len; ++i) {
139
+ unsigned char byte = (unsigned char)text[i];
140
+ if (byte == '"' || byte == '\\') {
141
+ char escaped[2] = {'\\', (char)byte};
142
+ code = buffer_write(
143
+ buffer, escaped, sizeof(escaped), limits, work, error);
144
+ } else if (byte < 0x20) {
145
+ char escaped[6] = {'\\', 'u', '0', '0', hex[byte >> 4], hex[byte & 15]};
146
+ code = buffer_write(
147
+ buffer, escaped, sizeof(escaped), limits, work, error);
148
+ } else {
149
+ code = buffer_write(buffer, &text[i], 1, limits, work, error);
150
+ }
151
+ if (code != TF_TRANSFORM_OK) return code;
152
+ }
153
+ return buffer_write(buffer, "\"", 1, limits, work, error);
154
+ }
155
+
156
+ static tf_transform_code compare_object_keys(
157
+ const cJSON *left, const cJSON *right,
158
+ const tf_transform_limits_v1 *limits, tf_json_work *work,
159
+ int *result, tf_transform_error **error) {
160
+ const unsigned char *a;
161
+ const unsigned char *b;
162
+ size_t compared = 0;
163
+ if (!left || !right || !left->string || !right->string || !result)
164
+ return tf_transform_set_error(
165
+ error, TF_TRANSFORM_CORRUPT_PLAN,
166
+ "canonical JSON object key is invalid");
167
+ a = (const unsigned char *)left->string;
168
+ b = (const unsigned char *)right->string;
169
+ while (compared <= limits->max_string_bytes && *a && *b && *a == *b) {
170
+ tf_transform_code code = json_work_progress(work, 1, 0, 0);
171
+ if (code != TF_TRANSFORM_OK) return code;
172
+ ++a;
173
+ ++b;
174
+ ++compared;
175
+ }
176
+ if (compared > limits->max_string_bytes)
177
+ return tf_transform_set_error(
178
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
179
+ "canonical JSON object key exceeds limit");
180
+ *result = *a < *b ? -1 : (*a > *b ? 1 : 0);
181
+ return TF_TRANSFORM_OK;
182
+ }
183
+
184
+ static tf_transform_code sort_object_items(
185
+ const cJSON **items, size_t count, tf_json_work *work,
186
+ tf_transform_error **error) {
187
+ size_t start;
188
+ size_t end;
189
+ (void)error;
190
+ if (count < 2) return TF_TRANSFORM_OK;
191
+ start = count / 2;
192
+ end = count;
193
+ for (;;) {
194
+ size_t root;
195
+ const cJSON *saved;
196
+ tf_transform_code code;
197
+ if (start != 0) {
198
+ --start;
199
+ root = start;
200
+ saved = items[root];
201
+ } else {
202
+ --end;
203
+ if (end == 0) break;
204
+ saved = items[end];
205
+ items[end] = items[0];
206
+ root = 0;
207
+ }
208
+ while (root <= (end - 1) / 2 && end > 1) {
209
+ size_t child = root * 2 + 1;
210
+ if (child + 1 < end) {
211
+ int compared;
212
+ code = json_work_progress(work, 0, 1, 0);
213
+ if (code != TF_TRANSFORM_OK) return code;
214
+ code = compare_object_keys(
215
+ items[child], items[child + 1],
216
+ &work->runtime->limits, work, &compared, error);
217
+ if (code != TF_TRANSFORM_OK) return code;
218
+ if (compared < 0)
219
+ ++child;
220
+ }
221
+ code = json_work_progress(work, 0, 1, 0);
222
+ if (code != TF_TRANSFORM_OK) return code;
223
+ {
224
+ int compared;
225
+ code = compare_object_keys(
226
+ saved, items[child], &work->runtime->limits,
227
+ work, &compared, error);
228
+ if (code != TF_TRANSFORM_OK) return code;
229
+ if (compared >= 0) break;
230
+ }
231
+ items[root] = items[child];
232
+ root = child;
233
+ }
234
+ items[root] = saved;
235
+ }
236
+ return TF_TRANSFORM_OK;
237
+ }
238
+
239
+ static tf_transform_code write_canonical_value(
240
+ const cJSON *value, tf_transform_buffer *buffer,
241
+ const tf_transform_limits_v1 *limits, size_t depth,
242
+ tf_json_work *work, tf_transform_error **error) {
243
+ tf_transform_code code;
244
+ if (!value || depth > limits->max_json_depth)
245
+ return tf_transform_set_error(
246
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON depth exceeds limit");
247
+ code = json_work_progress(work, 0, 1, 0);
248
+ if (code != TF_TRANSFORM_OK) return code;
249
+ if (cJSON_IsNull(value))
250
+ return buffer_write(buffer, "null", 4, limits, work, error);
251
+ if (cJSON_IsTrue(value))
252
+ return buffer_write(buffer, "true", 4, limits, work, error);
253
+ if (cJSON_IsFalse(value))
254
+ return buffer_write(buffer, "false", 5, limits, work, error);
255
+ if (cJSON_IsString(value))
256
+ return write_json_string(
257
+ buffer, value->valuestring, limits, work, error);
258
+ if (cJSON_IsNumber(value)) {
259
+ char number[32];
260
+ int count;
261
+ double integer;
262
+ if (!isfinite(value->valuedouble)
263
+ || modf(value->valuedouble, &integer) != 0.0
264
+ || integer < -9007199254740991.0
265
+ || integer > 9007199254740991.0)
266
+ return tf_transform_set_error(
267
+ error, TF_TRANSFORM_CORRUPT_PLAN,
268
+ "canonical JSON number is not a safe integer");
269
+ if (integer == 0.0) integer = 0.0;
270
+ count = snprintf(number, sizeof(number), "%.0f", integer);
271
+ if (count <= 0 || (size_t)count >= sizeof(number))
272
+ return tf_transform_set_error(
273
+ error, TF_TRANSFORM_INTERNAL, "canonical integer formatting failed");
274
+ return buffer_write(
275
+ buffer, number, (size_t)count, limits, work, error);
276
+ }
277
+ if (cJSON_IsArray(value)) {
278
+ const cJSON *child;
279
+ int first = 1;
280
+ code = buffer_write(buffer, "[", 1, limits, work, error);
281
+ if (code != TF_TRANSFORM_OK) return code;
282
+ cJSON_ArrayForEach(child, value) {
283
+ if (!first) {
284
+ code = buffer_write(buffer, ",", 1, limits, work, error);
285
+ if (code != TF_TRANSFORM_OK) return code;
286
+ }
287
+ first = 0;
288
+ code = write_canonical_value(
289
+ child, buffer, limits, depth + 1, work, error);
290
+ if (code != TF_TRANSFORM_OK) return code;
291
+ }
292
+ return buffer_write(buffer, "]", 1, limits, work, error);
293
+ }
294
+ if (cJSON_IsObject(value)) {
295
+ const cJSON *child;
296
+ const cJSON *count_child = value->child;
297
+ const cJSON **items;
298
+ size_t count = 0;
299
+ size_t index = 0;
300
+ cJSON_ArrayForEach(child, value) {
301
+ code = json_work_progress(work, 0, 1, 0);
302
+ if (code != TF_TRANSFORM_OK) return code;
303
+ ++count;
304
+ }
305
+ if (count > limits->max_object_keys
306
+ || count > SIZE_MAX / sizeof(items[0]))
307
+ return tf_transform_set_error(
308
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical object exceeds key limit");
309
+ if (count > limits->max_allocation_bytes / sizeof(items[0]))
310
+ return tf_transform_set_error(
311
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
312
+ "canonical key array exceeds allocation limit");
313
+ items = NULL;
314
+ if (!buffer->count_only && count != 0) {
315
+ uint64_t allocation_bytes = (uint64_t)count * sizeof(items[0]);
316
+ code = json_work_allocation(work, allocation_bytes, error);
317
+ if (code != TF_TRANSFORM_OK) return code;
318
+ items = (const cJSON **)malloc((size_t)allocation_bytes);
319
+ }
320
+ if (!buffer->count_only && count && !items) {
321
+ json_work_release(work, (uint64_t)count * sizeof(items[0]));
322
+ return tf_transform_set_error(
323
+ error, TF_TRANSFORM_ALLOCATION,
324
+ "canonical key allocation failed");
325
+ }
326
+ if (!buffer->count_only) {
327
+ cJSON_ArrayForEach(child, value) items[index++] = child;
328
+ code = sort_object_items(items, count, work, error);
329
+ if (code != TF_TRANSFORM_OK) goto object_done;
330
+ for (size_t i = 1; i < count; ++i) {
331
+ int compared;
332
+ code = compare_object_keys(
333
+ items[i - 1], items[i], limits,
334
+ work, &compared, error);
335
+ if (code != TF_TRANSFORM_OK) goto object_done;
336
+ if (compared == 0) {
337
+ code = tf_transform_set_error(
338
+ error, TF_TRANSFORM_CORRUPT_PLAN,
339
+ "duplicate JSON object key");
340
+ goto object_done;
341
+ }
342
+ }
343
+ }
344
+ code = buffer_write(buffer, "{", 1, limits, work, error);
345
+ for (size_t i = 0; code == TF_TRANSFORM_OK && i < count; ++i) {
346
+ const cJSON *item = buffer->count_only ? count_child : items[i];
347
+ if (!item) {
348
+ code = tf_transform_set_error(
349
+ error, TF_TRANSFORM_INTERNAL,
350
+ "canonical object traversal failed");
351
+ break;
352
+ }
353
+ if (i != 0)
354
+ code = buffer_write(buffer, ",", 1, limits, work, error);
355
+ if (code == TF_TRANSFORM_OK) code = write_json_string(
356
+ buffer, item->string, limits, work, error);
357
+ if (code == TF_TRANSFORM_OK)
358
+ code = buffer_write(buffer, ":", 1, limits, work, error);
359
+ if (code == TF_TRANSFORM_OK)
360
+ code = write_canonical_value(
361
+ item, buffer, limits, depth + 1, work, error);
362
+ if (buffer->count_only) count_child = item->next;
363
+ }
364
+ object_done:
365
+ free(items);
366
+ if (!buffer->count_only && count != 0)
367
+ json_work_release(work, (uint64_t)count * sizeof(items[0]));
368
+ if (code != TF_TRANSFORM_OK) return code;
369
+ return buffer_write(buffer, "}", 1, limits, work, error);
370
+ }
371
+ return tf_transform_set_error(
372
+ error, TF_TRANSFORM_CORRUPT_PLAN, "unsupported canonical JSON value");
373
+ }
374
+
375
+ static tf_transform_code json_print_canonical_common(
376
+ const cJSON *value, const tf_transform_runtime_copy *runtime,
377
+ tf_transform_resource_ledger *shared_ledger,
378
+ uint8_t **out, size_t *out_len, tf_transform_error **error) {
379
+ tf_transform_buffer count_buffer = {0};
380
+ tf_transform_buffer write_buffer = {0};
381
+ tf_json_work work = {0};
382
+ tf_transform_resource_ledger local_ledger;
383
+ tf_transform_code code;
384
+ if (out) *out = NULL;
385
+ if (out_len) *out_len = 0;
386
+ tf_transform_clear_error(error);
387
+ if (!value || !runtime || !out || !out_len)
388
+ return tf_transform_set_error(
389
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "canonical JSON argument is null");
390
+ work.runtime = runtime;
391
+ if (shared_ledger) work.ledger = shared_ledger;
392
+ else {
393
+ tf_transform_resource_ledger_init(&local_ledger, runtime);
394
+ work.ledger = &local_ledger;
395
+ }
396
+ work.error = error;
397
+ code = json_work_progress(&work, 0, 0, 1);
398
+ if (code != TF_TRANSFORM_OK) return code;
399
+ count_buffer.count_only = 1;
400
+ code = write_canonical_value(
401
+ value, &count_buffer, &runtime->limits, 1, &work, error);
402
+ if (code != TF_TRANSFORM_OK) return code;
403
+ if ((uint64_t)count_buffer.len > runtime->limits.max_plan_bytes
404
+ || (uint64_t)count_buffer.len > runtime->limits.max_allocation_bytes)
405
+ return tf_transform_set_error(
406
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
407
+ "canonical JSON output exceeds limits");
408
+ if (count_buffer.len != 0) {
409
+ code = json_work_allocation(&work, (uint64_t)count_buffer.len, error);
410
+ if (code != TF_TRANSFORM_OK) return code;
411
+ write_buffer.data = (uint8_t *)malloc(count_buffer.len);
412
+ if (!write_buffer.data) {
413
+ json_work_release(&work, (uint64_t)count_buffer.len);
414
+ return tf_transform_set_error(
415
+ error, TF_TRANSFORM_ALLOCATION,
416
+ "canonical JSON allocation failed");
417
+ }
418
+ write_buffer.cap = count_buffer.len;
419
+ }
420
+ code = write_canonical_value(
421
+ value, &write_buffer, &runtime->limits, 1, &work, error);
422
+ if (code != TF_TRANSFORM_OK) {
423
+ free(write_buffer.data);
424
+ json_work_release(&work, (uint64_t)count_buffer.len);
425
+ return code;
426
+ }
427
+ if (write_buffer.len != count_buffer.len) {
428
+ free(write_buffer.data);
429
+ json_work_release(&work, (uint64_t)count_buffer.len);
430
+ return tf_transform_set_error(
431
+ error, TF_TRANSFORM_INTERNAL,
432
+ "canonical JSON sizing pass drifted");
433
+ }
434
+ code = json_work_progress(&work, 0, 0, 1);
435
+ if (code != TF_TRANSFORM_OK) {
436
+ free(write_buffer.data);
437
+ json_work_release(&work, (uint64_t)count_buffer.len);
438
+ return code;
439
+ }
440
+ *out = write_buffer.data;
441
+ *out_len = write_buffer.len;
442
+ return TF_TRANSFORM_OK;
443
+ }
444
+
445
+ tf_transform_code tf_transform_json_print_canonical_runtime(
446
+ const cJSON *value, const tf_transform_runtime_copy *runtime,
447
+ uint8_t **out, size_t *out_len, tf_transform_error **error) {
448
+ return json_print_canonical_common(
449
+ value, runtime, NULL, out, out_len, error);
450
+ }
451
+
452
+ tf_transform_code tf_transform_json_print_canonical_runtime_ledger(
453
+ const cJSON *value, const tf_transform_runtime_copy *runtime,
454
+ tf_transform_resource_ledger *ledger,
455
+ uint8_t **out, size_t *out_len, tf_transform_error **error) {
456
+ if (!ledger || ledger->runtime != runtime) {
457
+ if (out) *out = NULL;
458
+ if (out_len) *out_len = 0;
459
+ return tf_transform_set_error(
460
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
461
+ "canonical JSON resource ledger is invalid");
462
+ }
463
+ return json_print_canonical_common(
464
+ value, runtime, ledger, out, out_len, error);
465
+ }
466
+
467
+ tf_transform_code tf_transform_json_print_canonical(
468
+ const cJSON *value, const tf_transform_limits_v1 *limits,
469
+ uint8_t **out, size_t *out_len, tf_transform_error **error) {
470
+ tf_transform_runtime_copy runtime;
471
+ if (!limits) {
472
+ if (out) *out = NULL;
473
+ if (out_len) *out_len = 0;
474
+ return tf_transform_set_error(
475
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
476
+ "canonical JSON limits are null");
477
+ }
478
+ memset(&runtime, 0, sizeof(runtime));
479
+ runtime.limits = *limits;
480
+ return json_print_canonical_common(
481
+ value, &runtime, NULL, out, out_len, error);
482
+ }
483
+
484
+ static int json_string_token_equals_ascii(
485
+ const uint8_t *token, size_t len, const char *expected) {
486
+ size_t input = 0;
487
+ size_t output = 0;
488
+ size_t expected_len = strlen(expected);
489
+ while (input < len) {
490
+ uint32_t decoded = token[input++];
491
+ if (decoded == '\\') {
492
+ unsigned char escaped;
493
+ if (input >= len) return 0;
494
+ escaped = token[input++];
495
+ if (escaped == 'u') {
496
+ decoded = 0;
497
+ if (len - input < 4) return 0;
498
+ for (size_t i = 0; i < 4; ++i) {
499
+ unsigned char byte = token[input++];
500
+ unsigned int nibble;
501
+ if (byte >= '0' && byte <= '9') nibble = byte - '0';
502
+ else if (byte >= 'a' && byte <= 'f')
503
+ nibble = byte - 'a' + 10u;
504
+ else if (byte >= 'A' && byte <= 'F')
505
+ nibble = byte - 'A' + 10u;
506
+ else return 0;
507
+ decoded = (decoded << 4) | nibble;
508
+ }
509
+ } else if (escaped == '"' || escaped == '\\' || escaped == '/') {
510
+ decoded = escaped;
511
+ } else if (escaped == 'b') decoded = '\b';
512
+ else if (escaped == 'f') decoded = '\f';
513
+ else if (escaped == 'n') decoded = '\n';
514
+ else if (escaped == 'r') decoded = '\r';
515
+ else if (escaped == 't') decoded = '\t';
516
+ else return 0;
517
+ }
518
+ if (decoded > UINT8_MAX || output >= expected_len
519
+ || (unsigned char)expected[output] != (unsigned char)decoded)
520
+ return 0;
521
+ ++output;
522
+ }
523
+ return output == expected_len;
524
+ }
525
+
526
+ static tf_transform_code scan_json_array_value(
527
+ size_t depth, const char *containers,
528
+ unsigned char *array_expects_value,
529
+ const unsigned char *category_arrays,
530
+ uint64_t *category_counts, size_t *array_elements,
531
+ uint64_t *total_categories, const tf_transform_limits_v1 *limits,
532
+ tf_transform_error **error) {
533
+ size_t index;
534
+ if (depth == 0 || containers[depth - 1] != '['
535
+ || !array_expects_value[depth - 1]) return TF_TRANSFORM_OK;
536
+ index = depth - 1;
537
+ if (*array_elements == SIZE_MAX)
538
+ return tf_transform_set_error(
539
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
540
+ "JSON array element count overflows");
541
+ ++*array_elements;
542
+ array_expects_value[index] = 0;
543
+ if (!category_arrays[index]) return TF_TRANSFORM_OK;
544
+ if (category_counts[index] == UINT64_MAX
545
+ || *total_categories == UINT64_MAX)
546
+ return tf_transform_set_error(
547
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
548
+ "JSON category count overflows");
549
+ ++category_counts[index];
550
+ ++*total_categories;
551
+ if (category_counts[index] > limits->max_categories_per_column
552
+ || *total_categories > limits->max_total_categories)
553
+ return tf_transform_set_error(
554
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
555
+ "JSON categories exceed configured limits before parsing");
556
+ return TF_TRANSFORM_OK;
557
+ }
558
+
559
+ static tf_transform_code scan_json_bounds(
560
+ const uint8_t *json, size_t len, const tf_transform_limits_v1 *limits,
561
+ tf_json_work *work, tf_json_scan_metrics *metrics,
562
+ tf_transform_error **error, tf_transform_code malformed_code) {
563
+ size_t depth = 0;
564
+ size_t keys = 0;
565
+ size_t strings = 0;
566
+ size_t current_string = 0;
567
+ size_t array_elements = 0;
568
+ size_t objects = 0;
569
+ size_t string_count = 0;
570
+ size_t number_count = 0;
571
+ size_t max_string = 0;
572
+ size_t current_number = 0;
573
+ size_t max_number = 0;
574
+ size_t last_string_start = 0;
575
+ size_t last_string_end = 0;
576
+ uint64_t total_categories = 0;
577
+ int in_string = 0;
578
+ int escaped = 0;
579
+ int in_number = 0;
580
+ int have_last_string = 0;
581
+ int expect_categories_array = 0;
582
+ char containers[CJSON_NESTING_LIMIT];
583
+ unsigned char array_expects_value[CJSON_NESTING_LIMIT];
584
+ unsigned char category_arrays[CJSON_NESTING_LIMIT];
585
+ uint64_t category_counts[CJSON_NESTING_LIMIT];
586
+ memset(category_arrays, 0, sizeof(category_arrays));
587
+ memset(category_counts, 0, sizeof(category_counts));
588
+ memset(metrics, 0, sizeof(*metrics));
589
+ for (size_t i = 0; i < len; ++i) {
590
+ uint8_t byte = json[i];
591
+ tf_transform_code progress = json_work_progress(work, 1, 0, 0);
592
+ if (progress != TF_TRANSFORM_OK) return progress;
593
+ if (byte == 0) return tf_transform_set_error(
594
+ error, malformed_code, "JSON contains an embedded null byte");
595
+ if (in_string) {
596
+ if (escaped) {
597
+ if (byte == 'u' && i + 4 < len
598
+ && json[i + 1] == '0' && json[i + 2] == '0'
599
+ && json[i + 3] == '0' && json[i + 4] == '0')
600
+ return tf_transform_set_error(
601
+ error, malformed_code,
602
+ "JSON strings may not contain U+0000");
603
+ ++current_string;
604
+ escaped = 0;
605
+ } else if (byte == '\\') {
606
+ ++current_string;
607
+ escaped = 1;
608
+ }
609
+ else if (byte == '"') {
610
+ in_string = 0;
611
+ if (string_count == SIZE_MAX) return tf_transform_set_error(
612
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
613
+ "JSON string count overflows");
614
+ ++string_count;
615
+ if (current_string > max_string) max_string = current_string;
616
+ if (current_string > limits->max_string_bytes)
617
+ return tf_transform_set_error(
618
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
619
+ "JSON string exceeds resource limit");
620
+ if (strings > SIZE_MAX - current_string)
621
+ return tf_transform_set_error(
622
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
623
+ "JSON string byte counter overflow");
624
+ strings += current_string;
625
+ if (strings > limits->max_decoded_string_bytes)
626
+ return tf_transform_set_error(
627
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
628
+ "JSON strings exceed resource limit");
629
+ last_string_end = i;
630
+ have_last_string = 1;
631
+ } else {
632
+ ++current_string;
633
+ }
634
+ continue;
635
+ }
636
+ if (in_number) {
637
+ if ((byte >= '0' && byte <= '9') || byte == '+' || byte == '-'
638
+ || byte == '.' || byte == 'e' || byte == 'E') {
639
+ if (current_number == SIZE_MAX) return tf_transform_set_error(
640
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
641
+ "JSON number length overflows");
642
+ ++current_number;
643
+ continue;
644
+ }
645
+ in_number = 0;
646
+ if (number_count == SIZE_MAX) return tf_transform_set_error(
647
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
648
+ "JSON number count overflows");
649
+ ++number_count;
650
+ if (current_number > max_number) max_number = current_number;
651
+ current_number = 0;
652
+ }
653
+ if (byte == '"') {
654
+ tf_transform_code code = scan_json_array_value(
655
+ depth, containers, array_expects_value,
656
+ category_arrays, category_counts, &array_elements,
657
+ &total_categories, limits, error);
658
+ if (code != TF_TRANSFORM_OK) return code;
659
+ expect_categories_array = 0;
660
+ have_last_string = 0;
661
+ in_string = 1;
662
+ current_string = 0;
663
+ last_string_start = i + 1;
664
+ } else if (byte == '{' || byte == '[') {
665
+ int is_category_array = byte == '[' && expect_categories_array;
666
+ tf_transform_code code = scan_json_array_value(
667
+ depth, containers, array_expects_value,
668
+ category_arrays, category_counts, &array_elements,
669
+ &total_categories, limits, error);
670
+ if (code != TF_TRANSFORM_OK) return code;
671
+ expect_categories_array = 0;
672
+ have_last_string = 0;
673
+ if (byte == '{') ++objects;
674
+ if (depth >= CJSON_NESTING_LIMIT
675
+ || ++depth > limits->max_json_depth)
676
+ return tf_transform_set_error(
677
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON depth exceeds limit");
678
+ containers[depth - 1] = (char)byte;
679
+ array_expects_value[depth - 1] = byte == '[' ? 1u : 0u;
680
+ category_arrays[depth - 1] = is_category_array ? 1u : 0u;
681
+ category_counts[depth - 1] = 0;
682
+ } else if (byte == '}' || byte == ']') {
683
+ expect_categories_array = 0;
684
+ have_last_string = 0;
685
+ if (depth == 0) return tf_transform_set_error(
686
+ error, malformed_code, "unbalanced JSON structure");
687
+ if ((byte == '}' && containers[depth - 1] != '{')
688
+ || (byte == ']' && containers[depth - 1] != '['))
689
+ return tf_transform_set_error(
690
+ error, malformed_code, "mismatched JSON structure");
691
+ --depth;
692
+ } else if (byte == ':') {
693
+ if (++keys > limits->max_object_keys)
694
+ return tf_transform_set_error(
695
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON key count exceeds limit");
696
+ expect_categories_array = have_last_string
697
+ && json_string_token_equals_ascii(
698
+ json + last_string_start,
699
+ last_string_end - last_string_start, "categories");
700
+ have_last_string = 0;
701
+ } else if (byte == ',') {
702
+ expect_categories_array = 0;
703
+ have_last_string = 0;
704
+ if (depth != 0 && containers[depth - 1] == '[')
705
+ array_expects_value[depth - 1] = 1;
706
+ } else if (byte == '-' || (byte >= '0' && byte <= '9')) {
707
+ tf_transform_code code = scan_json_array_value(
708
+ depth, containers, array_expects_value,
709
+ category_arrays, category_counts, &array_elements,
710
+ &total_categories, limits, error);
711
+ if (code != TF_TRANSFORM_OK) return code;
712
+ expect_categories_array = 0;
713
+ have_last_string = 0;
714
+ in_number = 1;
715
+ current_number = 1;
716
+ } else if (byte == 't' || byte == 'f' || byte == 'n') {
717
+ tf_transform_code code = scan_json_array_value(
718
+ depth, containers, array_expects_value,
719
+ category_arrays, category_counts, &array_elements,
720
+ &total_categories, limits, error);
721
+ if (code != TF_TRANSFORM_OK) return code;
722
+ expect_categories_array = 0;
723
+ have_last_string = 0;
724
+ } else if (byte != ' ' && byte != '\t'
725
+ && byte != '\r' && byte != '\n') {
726
+ expect_categories_array = 0;
727
+ have_last_string = 0;
728
+ }
729
+ }
730
+ if (in_number) {
731
+ ++number_count;
732
+ if (current_number > max_number) max_number = current_number;
733
+ }
734
+ if (in_string || depth != 0) return tf_transform_set_error(
735
+ error, malformed_code, "unterminated JSON structure");
736
+ if (max_number > 64) return tf_transform_set_error(
737
+ error, malformed_code,
738
+ "JSON numeric token exceeds bounded safe-integer syntax");
739
+ {
740
+ uint64_t nodes = 1;
741
+ uint64_t allocation_count;
742
+ uint64_t resident;
743
+ uint64_t persistent;
744
+ if ((uint64_t)keys > UINT64_MAX - nodes) goto metric_overflow;
745
+ nodes += (uint64_t)keys;
746
+ if ((uint64_t)array_elements > UINT64_MAX - nodes) goto metric_overflow;
747
+ nodes += (uint64_t)array_elements;
748
+ allocation_count = nodes;
749
+ if ((uint64_t)string_count > UINT64_MAX - allocation_count)
750
+ goto metric_overflow;
751
+ allocation_count += (uint64_t)string_count;
752
+ if ((uint64_t)number_count > UINT64_MAX - allocation_count)
753
+ goto metric_overflow;
754
+ allocation_count += (uint64_t)number_count;
755
+ if (nodes > UINT64_MAX / sizeof(cJSON)) goto metric_overflow;
756
+ persistent = nodes * sizeof(cJSON);
757
+ if ((uint64_t)strings > UINT64_MAX - persistent) goto metric_overflow;
758
+ persistent += (uint64_t)strings;
759
+ if ((uint64_t)string_count > UINT64_MAX - persistent)
760
+ goto metric_overflow;
761
+ persistent += (uint64_t)string_count;
762
+ resident = persistent;
763
+ if (number_count != 0
764
+ && (uint64_t)max_number + 1 > UINT64_MAX - resident)
765
+ goto metric_overflow;
766
+ if (number_count != 0) resident += (uint64_t)max_number + 1;
767
+ if (allocation_count > limits->max_allocations_per_session
768
+ || resident > limits->max_resident_state_bytes
769
+ || sizeof(cJSON) > limits->max_allocation_bytes
770
+ || (string_count != 0
771
+ && (uint64_t)max_string + 1 > limits->max_allocation_bytes)
772
+ || (number_count != 0
773
+ && (uint64_t)max_number + 1 > limits->max_allocation_bytes))
774
+ return tf_transform_set_error(
775
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
776
+ "JSON parser growth exceeds session limits");
777
+ metrics->node_count_upper = nodes;
778
+ metrics->string_count = string_count;
779
+ metrics->number_count = number_count;
780
+ metrics->object_count = objects;
781
+ metrics->decoded_string_bytes = strings;
782
+ metrics->max_string_bytes = max_string;
783
+ metrics->max_number_bytes = max_number;
784
+ metrics->resident_bytes_persistent = persistent;
785
+ metrics->resident_bytes_upper = resident;
786
+ metrics->allocation_count_upper = allocation_count;
787
+ }
788
+ return TF_TRANSFORM_OK;
789
+ metric_overflow:
790
+ return tf_transform_set_error(
791
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
792
+ "JSON parser resource estimate overflows");
793
+ }
794
+
795
+ static int json_valid_utf8_with_work(
796
+ const uint8_t *data, size_t len, tf_json_work *work) {
797
+ size_t i = 0;
798
+ while (i < len) {
799
+ uint8_t c = data[i++];
800
+ uint32_t code;
801
+ size_t needed;
802
+ if (json_work_progress(work, 1, 0, 0) != TF_TRANSFORM_OK) return -1;
803
+ if (c <= 0x7f) {
804
+ if (c == 0) return 0;
805
+ continue;
806
+ }
807
+ if (c >= 0xc2 && c <= 0xdf) {
808
+ code = (uint32_t)(c & 0x1f);
809
+ needed = 1;
810
+ } else if (c >= 0xe0 && c <= 0xef) {
811
+ code = (uint32_t)(c & 0x0f);
812
+ needed = 2;
813
+ } else if (c >= 0xf0 && c <= 0xf4) {
814
+ code = (uint32_t)(c & 0x07);
815
+ needed = 3;
816
+ } else return 0;
817
+ if (needed > len - i) return 0;
818
+ for (size_t j = 0; j < needed; ++j) {
819
+ uint8_t next = data[i++];
820
+ if (json_work_progress(work, 1, 0, 0) != TF_TRANSFORM_OK) return -1;
821
+ if ((next & 0xc0) != 0x80) return 0;
822
+ code = (code << 6) | (uint32_t)(next & 0x3f);
823
+ }
824
+ if ((needed == 1 && code < 0x80)
825
+ || (needed == 2 && code < 0x800)
826
+ || (needed == 3 && code < 0x10000)
827
+ || code > 0x10ffff
828
+ || (code >= 0xd800 && code <= 0xdfff)) return 0;
829
+ }
830
+ return 1;
831
+ }
832
+
833
+ static tf_transform_code reject_duplicate_keys(
834
+ const cJSON *value, size_t depth, const tf_transform_limits_v1 *limits,
835
+ tf_json_work *work, tf_transform_error **error,
836
+ tf_transform_code malformed_code) {
837
+ const cJSON *child;
838
+ tf_transform_code code;
839
+ if (depth > limits->max_json_depth) return tf_transform_set_error(
840
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON depth exceeds limit");
841
+ if (cJSON_IsObject(value)) {
842
+ const cJSON **items = NULL;
843
+ size_t count = 0;
844
+ size_t index = 0;
845
+ cJSON_ArrayForEach(child, value) ++count;
846
+ if (count > 1) {
847
+ uint64_t bytes = (uint64_t)count * sizeof(items[0]);
848
+ code = json_work_allocation(work, bytes, error);
849
+ if (code != TF_TRANSFORM_OK) return code;
850
+ items = (const cJSON **)malloc((size_t)bytes);
851
+ if (!items) {
852
+ json_work_release(work, bytes);
853
+ return tf_transform_set_error(
854
+ error, TF_TRANSFORM_ALLOCATION,
855
+ "duplicate-key index allocation failed");
856
+ }
857
+ cJSON_ArrayForEach(child, value) items[index++] = child;
858
+ code = sort_object_items(items, count, work, error);
859
+ if (code == TF_TRANSFORM_OK) {
860
+ for (size_t i = 1; i < count; ++i) {
861
+ int compared;
862
+ code = json_work_progress(work, 0, 1, 0);
863
+ if (code != TF_TRANSFORM_OK) break;
864
+ code = compare_object_keys(
865
+ items[i - 1], items[i], limits,
866
+ work, &compared, error);
867
+ if (code != TF_TRANSFORM_OK) break;
868
+ if (compared == 0) {
869
+ code = tf_transform_set_error(
870
+ error, malformed_code,
871
+ "duplicate JSON object key");
872
+ break;
873
+ }
874
+ }
875
+ }
876
+ free(items);
877
+ json_work_release(work, bytes);
878
+ if (code != TF_TRANSFORM_OK) return code;
879
+ }
880
+ }
881
+ cJSON_ArrayForEach(child, value) {
882
+ code = json_work_progress(work, 0, 1, 0);
883
+ if (code != TF_TRANSFORM_OK) return code;
884
+ code = reject_duplicate_keys(
885
+ child, depth + 1, limits, work, error, malformed_code);
886
+ if (code != TF_TRANSFORM_OK) return code;
887
+ }
888
+ return TF_TRANSFORM_OK;
889
+ }
890
+
891
+ static void delete_json_runtime_walk(
892
+ cJSON *item, const tf_transform_runtime_copy *runtime,
893
+ size_t *iterations, int *cancelled) {
894
+ while (item) {
895
+ cJSON *next = item->next;
896
+ if (!(item->type & cJSON_IsReference) && item->child)
897
+ delete_json_runtime_walk(
898
+ item->child, runtime, iterations, cancelled);
899
+ if (!(item->type & cJSON_IsReference) && item->valuestring)
900
+ cJSON_free(item->valuestring);
901
+ if (!(item->type & cJSON_StringIsConst) && item->string)
902
+ cJSON_free(item->string);
903
+ cJSON_free(item);
904
+ ++*iterations;
905
+ if (*iterations % TF_TRANSFORM_CANCEL_ITERS_V1 == 0
906
+ && runtime && runtime->cancel
907
+ && runtime->cancel(runtime->cancel_user))
908
+ *cancelled = 1;
909
+ item = next;
910
+ }
911
+ }
912
+
913
+ static int delete_json_runtime(
914
+ cJSON *item, const tf_transform_runtime_copy *runtime) {
915
+ size_t iterations = 0;
916
+ int cancelled = 0;
917
+ if (runtime && runtime->cancel && runtime->cancel(runtime->cancel_user))
918
+ cancelled = 1;
919
+ delete_json_runtime_walk(item, runtime, &iterations, &cancelled);
920
+ if (runtime && runtime->cancel && runtime->cancel(runtime->cancel_user))
921
+ cancelled = 1;
922
+ return cancelled;
923
+ }
924
+
925
+ static tf_transform_code json_parse_bounded_common(
926
+ const uint8_t *json, size_t json_len,
927
+ const tf_transform_runtime_copy *runtime,
928
+ tf_transform_resource_ledger *shared_ledger, cJSON **out,
929
+ uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
930
+ tf_transform_error **error, tf_transform_code malformed_code) {
931
+ cJSON *value;
932
+ const char *parse_end = NULL;
933
+ tf_transform_code code;
934
+ tf_json_work work = {0};
935
+ tf_transform_resource_ledger local_ledger;
936
+ tf_transform_resource_ledger *ledger;
937
+ tf_json_scan_metrics metrics;
938
+ uint64_t allocation_start;
939
+ int valid_utf8;
940
+ if (out) *out = NULL;
941
+ if (resident_bytes_out) *resident_bytes_out = 0;
942
+ if (allocation_count_out) *allocation_count_out = 0;
943
+ if (!json || json_len == 0 || !runtime || !out)
944
+ return tf_transform_set_error(
945
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "JSON argument is null or empty");
946
+ work.runtime = runtime;
947
+ if (shared_ledger) ledger = shared_ledger;
948
+ else {
949
+ tf_transform_resource_ledger_init(&local_ledger, runtime);
950
+ ledger = &local_ledger;
951
+ }
952
+ work.ledger = ledger;
953
+ work.error = error;
954
+ allocation_start = ledger->allocation_count;
955
+ code = json_work_progress(&work, 0, 0, 1);
956
+ if (code != TF_TRANSFORM_OK) return code;
957
+ if (json_len > runtime->limits.max_plan_bytes || json_len > SIZE_MAX - 1)
958
+ return tf_transform_set_error(
959
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON input exceeds byte limit");
960
+ valid_utf8 = json_valid_utf8_with_work(json, json_len, &work);
961
+ if (valid_utf8 < 0) return TF_TRANSFORM_CANCELLED;
962
+ if (!valid_utf8)
963
+ return tf_transform_set_error(
964
+ error, malformed_code, "JSON is not valid UTF-8");
965
+ code = scan_json_bounds(
966
+ json, json_len, &runtime->limits, &work, &metrics,
967
+ error, malformed_code);
968
+ if (code != TF_TRANSFORM_OK) return code;
969
+ code = tf_transform_resource_charge_batch(
970
+ ledger, metrics.allocation_count_upper,
971
+ metrics.resident_bytes_persistent, metrics.resident_bytes_upper,
972
+ error);
973
+ if (code != TF_TRANSFORM_OK) return code;
974
+ value = cJSON_ParseWithLengthOptsAndPoll(
975
+ (const char *)json, json_len, &parse_end, 0,
976
+ json_parser_poll, &work, TF_TRANSFORM_CANCEL_BYTES_V1,
977
+ TF_TRANSFORM_CANCEL_ITERS_V1);
978
+ if (!value && work.cancelled) return tf_transform_set_error(
979
+ error, TF_TRANSFORM_CANCELLED, "prepared transform cancelled");
980
+ if (!value) return tf_transform_set_error(
981
+ error, cJSON_ParseHadAllocationFailure()
982
+ ? TF_TRANSFORM_ALLOCATION : malformed_code,
983
+ cJSON_ParseHadAllocationFailure()
984
+ ? "JSON parser allocation failed" : "invalid JSON");
985
+ while ((size_t)(parse_end - (const char *)json) < json_len
986
+ && (*parse_end == ' ' || *parse_end == '\t'
987
+ || *parse_end == '\r' || *parse_end == '\n')) {
988
+ code = json_work_progress(&work, 1, 0, 0);
989
+ if (code != TF_TRANSFORM_OK) {
990
+ (void)delete_json_runtime(value, runtime);
991
+ return code;
992
+ }
993
+ ++parse_end;
994
+ }
995
+ if ((size_t)(parse_end - (const char *)json) != json_len) {
996
+ (void)delete_json_runtime(value, runtime);
997
+ return tf_transform_set_error(
998
+ error, malformed_code, "JSON contains trailing data");
999
+ }
1000
+ code = reject_duplicate_keys(
1001
+ value, 1, &runtime->limits, &work, error, malformed_code);
1002
+ if (code != TF_TRANSFORM_OK) {
1003
+ (void)delete_json_runtime(value, runtime);
1004
+ return code;
1005
+ }
1006
+ code = json_work_progress(&work, 0, 0, 1);
1007
+ if (code != TF_TRANSFORM_OK) {
1008
+ (void)delete_json_runtime(value, runtime);
1009
+ return code;
1010
+ }
1011
+ *out = value;
1012
+ if (resident_bytes_out)
1013
+ *resident_bytes_out = metrics.resident_bytes_persistent;
1014
+ if (allocation_count_out)
1015
+ *allocation_count_out = ledger->allocation_count - allocation_start;
1016
+ return TF_TRANSFORM_OK;
1017
+ }
1018
+
1019
+ tf_transform_code tf_transform_json_parse_bounded_runtime(
1020
+ const uint8_t *json, size_t json_len,
1021
+ const tf_transform_runtime_copy *runtime, cJSON **out,
1022
+ uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
1023
+ tf_transform_error **error, tf_transform_code malformed_code) {
1024
+ return json_parse_bounded_common(
1025
+ json, json_len, runtime, NULL, out,
1026
+ resident_bytes_out, allocation_count_out,
1027
+ error, malformed_code);
1028
+ }
1029
+
1030
+ tf_transform_code tf_transform_json_parse_bounded_runtime_ledger(
1031
+ const uint8_t *json, size_t json_len,
1032
+ const tf_transform_runtime_copy *runtime,
1033
+ tf_transform_resource_ledger *ledger, cJSON **out,
1034
+ uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
1035
+ tf_transform_error **error, tf_transform_code malformed_code) {
1036
+ if (!ledger || ledger->runtime != runtime) {
1037
+ if (out) *out = NULL;
1038
+ return tf_transform_set_error(
1039
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
1040
+ "JSON parser resource ledger is invalid");
1041
+ }
1042
+ return json_parse_bounded_common(
1043
+ json, json_len, runtime, ledger, out,
1044
+ resident_bytes_out, allocation_count_out,
1045
+ error, malformed_code);
1046
+ }
1047
+
1048
+ tf_transform_code tf_transform_json_parse_bounded(
1049
+ const uint8_t *json, size_t json_len,
1050
+ const tf_transform_limits_v1 *limits, cJSON **out,
1051
+ tf_transform_error **error, tf_transform_code malformed_code) {
1052
+ tf_transform_runtime_copy runtime;
1053
+ if (!limits) {
1054
+ if (out) *out = NULL;
1055
+ return tf_transform_set_error(
1056
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "JSON limits are null");
1057
+ }
1058
+ memset(&runtime, 0, sizeof(runtime));
1059
+ runtime.limits = *limits;
1060
+ return json_parse_bounded_common(
1061
+ json, json_len, &runtime, NULL, out, NULL, NULL,
1062
+ error, malformed_code);
1063
+ }
1064
+
1065
+ static int exact_keys(const cJSON *object, const char *const *keys, size_t count) {
1066
+ const cJSON *child;
1067
+ size_t seen = 0;
1068
+ if (!cJSON_IsObject(object)) return 0;
1069
+ cJSON_ArrayForEach(child, object) {
1070
+ int found = 0;
1071
+ for (size_t i = 0; i < count; ++i) {
1072
+ if (strcmp(child->string, keys[i]) == 0) { found = 1; break; }
1073
+ }
1074
+ if (!found) return 0;
1075
+ ++seen;
1076
+ }
1077
+ return seen == count;
1078
+ }
1079
+
1080
+ static const cJSON *required_item(const cJSON *object, const char *name) {
1081
+ return cJSON_GetObjectItemCaseSensitive(object, name);
1082
+ }
1083
+
1084
+ static int json_string_equals(const cJSON *value, const char *expected) {
1085
+ return cJSON_IsString(value) && value->valuestring
1086
+ && strcmp(value->valuestring, expected) == 0;
1087
+ }
1088
+
1089
+ static int safe_uint64(const cJSON *value, uint64_t *out) {
1090
+ double integer;
1091
+ if (!cJSON_IsNumber(value) || !isfinite(value->valuedouble)
1092
+ || value->valuedouble < 0
1093
+ || modf(value->valuedouble, &integer) != 0.0
1094
+ || integer >= 9007199254740992.0) return 0;
1095
+ *out = (uint64_t)integer;
1096
+ return 1;
1097
+ }
1098
+
1099
+ static int safe_int64(const cJSON *value, int64_t *out) {
1100
+ double integer;
1101
+ if (!cJSON_IsNumber(value) || !isfinite(value->valuedouble)
1102
+ || modf(value->valuedouble, &integer) != 0.0
1103
+ || integer <= -9007199254740992.0
1104
+ || integer >= 9007199254740992.0) return 0;
1105
+ *out = (int64_t)integer;
1106
+ return 1;
1107
+ }
1108
+
1109
+ static tf_transform_code runtime_cstring_length(
1110
+ const char *text, size_t maximum,
1111
+ const tf_transform_runtime_copy *runtime,
1112
+ size_t *out, tf_transform_error **error) {
1113
+ size_t length = 0;
1114
+ if (!text || !runtime || !out)
1115
+ return tf_transform_set_error(
1116
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
1117
+ "runtime string argument is null");
1118
+ while (length <= maximum && text[length] != '\0') {
1119
+ if (length % TF_TRANSFORM_CANCEL_BYTES_V1 == 0) {
1120
+ tf_transform_code code = tf_transform_poll_cancel(runtime, error);
1121
+ if (code != TF_TRANSFORM_OK) return code;
1122
+ }
1123
+ ++length;
1124
+ }
1125
+ if (length > maximum)
1126
+ return tf_transform_set_error(
1127
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
1128
+ "runtime string exceeds byte limit");
1129
+ *out = length;
1130
+ return tf_transform_poll_cancel(runtime, error);
1131
+ }
1132
+
1133
+ static tf_transform_code runtime_span_equal(
1134
+ const char *left, size_t left_len,
1135
+ const char *right, size_t right_len,
1136
+ const tf_transform_runtime_copy *runtime,
1137
+ int *equal, tf_transform_error **error) {
1138
+ int comparison = 0;
1139
+ tf_transform_code code;
1140
+ if (!left || !right || !runtime || !equal)
1141
+ return tf_transform_set_error(
1142
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
1143
+ "runtime span argument is null");
1144
+ code = tf_transform_compare_bytes_runtime(
1145
+ left, left_len, right, right_len,
1146
+ runtime, &comparison, error);
1147
+ if (code != TF_TRANSFORM_OK) return code;
1148
+ *equal = comparison == 0;
1149
+ return TF_TRANSFORM_OK;
1150
+ }
1151
+
1152
+ static int parse_tagged_bits(
1153
+ const cJSON *value, uint64_t *out, uint32_t *dtype) {
1154
+ static const char *const keys[] = {"t", "v"};
1155
+ const cJSON *tag;
1156
+ const cJSON *bits;
1157
+ uint64_t decoded = 0;
1158
+ size_t digits;
1159
+ if (!out || !dtype || !exact_keys(value, keys, 2)) return 0;
1160
+ tag = required_item(value, "t");
1161
+ bits = required_item(value, "v");
1162
+ if (!cJSON_IsString(tag) || !tag->valuestring
1163
+ || !cJSON_IsString(bits) || !bits->valuestring) return 0;
1164
+ if (strcmp(tag->valuestring, "f64") == 0) {
1165
+ *dtype = TF_VIEW_FLOAT64;
1166
+ digits = 16;
1167
+ } else if (strcmp(tag->valuestring, "f32") == 0) {
1168
+ *dtype = TF_VIEW_FLOAT32;
1169
+ digits = 8;
1170
+ } else return 0;
1171
+ for (size_t i = 0; i < digits; ++i) {
1172
+ unsigned char c = (unsigned char)bits->valuestring[i];
1173
+ unsigned int nibble;
1174
+ if (c == '\0') return 0;
1175
+ if (c >= '0' && c <= '9') nibble = c - '0';
1176
+ else if (c >= 'a' && c <= 'f') nibble = c - 'a' + 10u;
1177
+ else return 0;
1178
+ decoded = (decoded << 4) | nibble;
1179
+ }
1180
+ if (bits->valuestring[digits] != '\0') return 0;
1181
+ *out = decoded;
1182
+ return 1;
1183
+ }
1184
+
1185
+ static int parse_tagged_constant(const cJSON *value, double *out, uint32_t *dtype) {
1186
+ uint64_t decoded = 0;
1187
+ if (!out || !parse_tagged_bits(value, &decoded, dtype)) return 0;
1188
+ if (*dtype == TF_VIEW_FLOAT64) *out = tf_transform_double_from_bits(decoded);
1189
+ else {
1190
+ uint32_t fbits = (uint32_t)decoded;
1191
+ float fvalue;
1192
+ memcpy(&fvalue, &fbits, sizeof(fvalue));
1193
+ *out = (double)fvalue;
1194
+ }
1195
+ return tf_transform_double_is_finite(*out);
1196
+ }
1197
+
1198
+ static int parse_numeric_recipe_branch(
1199
+ const cJSON *numeric, tf_transform_recipe_column *out) {
1200
+ static const char *const numeric_keys[] = {"impute", "normalize"};
1201
+ static const char *const impute_keys[] = {"allMissing", "constant", "op"};
1202
+ static const char *const normalize_keys[] = {"ddof", "op"};
1203
+ const cJSON *impute;
1204
+ const cJSON *normalize;
1205
+ const cJSON *op;
1206
+ const cJSON *policy;
1207
+ const cJSON *constant;
1208
+ if (!exact_keys(numeric, numeric_keys, 2)) return 0;
1209
+ impute = required_item(numeric, "impute");
1210
+ if (!exact_keys(impute, impute_keys, 3)) return 0;
1211
+ op = required_item(impute, "op");
1212
+ policy = required_item(impute, "allMissing");
1213
+ constant = required_item(impute, "constant");
1214
+ if (json_string_equals(op, "none")) {
1215
+ if (!cJSON_IsNull(policy) || !cJSON_IsNull(constant)) return 0;
1216
+ out->impute = TF_TRANSFORM_IMPUTE_NONE;
1217
+ } else if (json_string_equals(op, "zero")) {
1218
+ if (!cJSON_IsNull(policy) || !cJSON_IsNull(constant)) return 0;
1219
+ out->impute = TF_TRANSFORM_IMPUTE_ZERO;
1220
+ out->constant = 0.0;
1221
+ } else if (json_string_equals(op, "constant")) {
1222
+ if (!cJSON_IsNull(policy)
1223
+ || !parse_tagged_constant(
1224
+ constant, &out->constant, &out->constant_dtype)) return 0;
1225
+ out->impute = TF_TRANSFORM_IMPUTE_CONSTANT;
1226
+ } else if (json_string_equals(op, "mean")
1227
+ || json_string_equals(op, "median")) {
1228
+ if (!cJSON_IsNull(constant)) return 0;
1229
+ if (json_string_equals(policy, "error"))
1230
+ out->all_missing = TF_TRANSFORM_ALL_MISSING_ERROR;
1231
+ else if (json_string_equals(policy, "zero"))
1232
+ out->all_missing = TF_TRANSFORM_ALL_MISSING_ZERO;
1233
+ else return 0;
1234
+ out->impute = json_string_equals(op, "mean")
1235
+ ? TF_TRANSFORM_IMPUTE_MEAN : TF_TRANSFORM_IMPUTE_MEDIAN;
1236
+ } else return 0;
1237
+ normalize = required_item(numeric, "normalize");
1238
+ if (!exact_keys(normalize, normalize_keys, 2)) return 0;
1239
+ op = required_item(normalize, "op");
1240
+ if (json_string_equals(op, "none")) {
1241
+ if (!cJSON_IsNull(required_item(normalize, "ddof"))) return 0;
1242
+ out->normalize = TF_TRANSFORM_NORMALIZE_NONE;
1243
+ } else if (json_string_equals(op, "minmax")) {
1244
+ if (!cJSON_IsNull(required_item(normalize, "ddof"))) return 0;
1245
+ out->normalize = TF_TRANSFORM_NORMALIZE_MINMAX;
1246
+ } else if (json_string_equals(op, "standard")) {
1247
+ uint64_t ddof;
1248
+ if (!safe_uint64(required_item(normalize, "ddof"), &ddof) || ddof > 1)
1249
+ return 0;
1250
+ out->normalize = TF_TRANSFORM_NORMALIZE_STANDARD;
1251
+ out->ddof = (int)ddof;
1252
+ } else return 0;
1253
+ return 1;
1254
+ }
1255
+
1256
+ static int parse_categorical_recipe_branch(
1257
+ const cJSON *categorical, tf_transform_recipe_column *out) {
1258
+ static const char *const categorical_keys[] = {"encode", "impute"};
1259
+ static const char *const impute_keys[] = {"allMissing", "constant", "op"};
1260
+ static const char *const encode_keys[] = {
1261
+ "categories", "op", "sentinelLabel", "unknown"
1262
+ };
1263
+ const cJSON *impute;
1264
+ const cJSON *encode;
1265
+ const cJSON *categories;
1266
+ const cJSON *impute_op;
1267
+ const cJSON *policy;
1268
+ if (!exact_keys(categorical, categorical_keys, 2)) return 0;
1269
+ impute = required_item(categorical, "impute");
1270
+ encode = required_item(categorical, "encode");
1271
+ if (!exact_keys(impute, impute_keys, 3)
1272
+ || !cJSON_IsNull(required_item(impute, "constant"))) return 0;
1273
+ impute_op = required_item(impute, "op");
1274
+ policy = required_item(impute, "allMissing");
1275
+ if (json_string_equals(impute_op, "none")) {
1276
+ if (!cJSON_IsNull(policy)) return 0;
1277
+ out->categorical_impute = TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE;
1278
+ out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_NONE;
1279
+ } else if (json_string_equals(impute_op, "mode")) {
1280
+ if (json_string_equals(policy, "error"))
1281
+ out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_ERROR;
1282
+ else if (json_string_equals(policy, "zero"))
1283
+ out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_ZERO;
1284
+ else return 0;
1285
+ out->categorical_impute = TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE;
1286
+ } else return 0;
1287
+ if (!exact_keys(encode, encode_keys, 4)) return 0;
1288
+ categories = required_item(encode, "categories");
1289
+ if (json_string_equals(required_item(encode, "op"), "none")) {
1290
+ if (!cJSON_IsNull(required_item(encode, "sentinelLabel"))
1291
+ || !cJSON_IsNull(required_item(encode, "unknown"))) return 0;
1292
+ out->categorical_encode = TF_TRANSFORM_ENCODE_NONE;
1293
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_NONE;
1294
+ } else if (json_string_equals(required_item(encode, "op"), "label")) {
1295
+ const cJSON *unknown = required_item(encode, "unknown");
1296
+ const cJSON *sentinel = required_item(encode, "sentinelLabel");
1297
+ out->categorical_encode = TF_TRANSFORM_ENCODE_LABEL;
1298
+ if (json_string_equals(unknown, "error") && cJSON_IsNull(sentinel))
1299
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ERROR;
1300
+ else if (json_string_equals(unknown, "other") && cJSON_IsNull(sentinel))
1301
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_OTHER;
1302
+ else if (json_string_equals(unknown, "sentinel")
1303
+ && safe_int64(sentinel, &out->categorical_sentinel_label)) {
1304
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_SENTINEL;
1305
+ out->categorical_has_sentinel_label = 1;
1306
+ } else return 0;
1307
+ } else if (json_string_equals(required_item(encode, "op"), "onehot")) {
1308
+ const cJSON *unknown = required_item(encode, "unknown");
1309
+ if (!cJSON_IsNull(required_item(encode, "sentinelLabel"))) return 0;
1310
+ out->categorical_encode = TF_TRANSFORM_ENCODE_ONEHOT;
1311
+ if (json_string_equals(unknown, "error"))
1312
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ERROR;
1313
+ else if (json_string_equals(unknown, "all_zero"))
1314
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ALL_ZERO;
1315
+ else if (json_string_equals(unknown, "other"))
1316
+ out->categorical_unknown = TF_TRANSFORM_UNKNOWN_OTHER;
1317
+ else return 0;
1318
+ } else return 0;
1319
+ if (out->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
1320
+ && out->categorical_encode == TF_TRANSFORM_ENCODE_NONE) {
1321
+ if (!cJSON_IsNull(categories)) return 0;
1322
+ out->categorical_discover = 0;
1323
+ } else {
1324
+ if (!json_string_equals(categories, "discover") && !cJSON_IsArray(categories)) return 0;
1325
+ out->categorical_discover = !cJSON_IsArray(categories);
1326
+ }
1327
+ return 1;
1328
+ }
1329
+
1330
+ static int tagged_category_bits(
1331
+ const cJSON *value, uint32_t expected_dtype, uint64_t *out);
1332
+
1333
+ static tf_transform_code parse_fixed_categories(
1334
+ const cJSON *categories, const tf_transform_runtime_copy *runtime,
1335
+ tf_transform_resource_ledger *ledger, tf_transform_recipe_column *out,
1336
+ tf_transform_error **error) {
1337
+ size_t count = 0;
1338
+ uint32_t dtype;
1339
+ uint64_t previous = 0;
1340
+ const cJSON *entry;
1341
+ tf_transform_code code;
1342
+ if (out->kind != TF_TRANSFORM_KIND_CATEGORICAL || !categories->child)
1343
+ goto invalid;
1344
+ dtype = json_string_equals(required_item(categories->child, "t"), "f32")
1345
+ ? TF_VIEW_FLOAT32 : TF_VIEW_FLOAT64;
1346
+ cJSON_ArrayForEach(entry, categories) {
1347
+ uint64_t bits;
1348
+ if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
1349
+ code = tf_transform_poll_cancel(runtime, error);
1350
+ if (code != TF_TRANSFORM_OK) return code;
1351
+ }
1352
+ if (!tagged_category_bits(entry, dtype, &bits)
1353
+ || (count && tf_transform_category_compare(previous, bits, dtype) >= 0))
1354
+ goto invalid;
1355
+ previous = bits;
1356
+ if ((uint64_t)count >= runtime->limits.max_categories_per_column
1357
+ || (uint64_t)count >= runtime->limits.max_total_categories
1358
+ || count >= SIZE_MAX / sizeof(*out->categorical_fixed))
1359
+ return tf_transform_set_error(error, TF_TRANSFORM_RESOURCE_LIMIT,
1360
+ "fixed dictionary exceeds category limits");
1361
+ count++;
1362
+ }
1363
+ out->categorical_fixed = tf_transform_resource_calloc(
1364
+ ledger, count, sizeof(*out->categorical_fixed), error);
1365
+ if (!out->categorical_fixed)
1366
+ return ledger->last_code != TF_TRANSFORM_OK ? ledger->last_code
1367
+ : tf_transform_set_error(error, TF_TRANSFORM_ALLOCATION,
1368
+ "fixed dictionary allocation failed");
1369
+ out->categorical_fixed_count = count;
1370
+ out->categorical_fixed_dtype = dtype;
1371
+ size_t i = 0;
1372
+ cJSON_ArrayForEach(entry, categories) {
1373
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
1374
+ code = tf_transform_poll_cancel(runtime, error);
1375
+ if (code != TF_TRANSFORM_OK) return code;
1376
+ }
1377
+ if (!tagged_category_bits(entry, dtype, &out->categorical_fixed[i++].bits))
1378
+ goto invalid;
1379
+ }
1380
+ return TF_TRANSFORM_OK;
1381
+ invalid:
1382
+ return tf_transform_set_error(error, TF_TRANSFORM_INVALID_RECIPE,
1383
+ "fixed categories require a declared categorical column and sorted unique finite matching tags");
1384
+ }
1385
+
1386
+ static tf_transform_code parse_recipe_column(
1387
+ const cJSON *value, const tf_transform_runtime_copy *runtime,
1388
+ tf_transform_resource_ledger *ledger,
1389
+ tf_transform_recipe_column *out, tf_transform_error **error) {
1390
+ static const char *const column_keys[] = {
1391
+ "categorical", "kind", "numeric", "sourceId"
1392
+ };
1393
+ static const char *const kind_keys[] = {
1394
+ "maxCategories", "op", "rule", "value"
1395
+ };
1396
+ const cJSON *kind;
1397
+ const cJSON *numeric;
1398
+ const cJSON *categorical;
1399
+ const cJSON *source_id;
1400
+ const cJSON *kind_op;
1401
+ const cJSON *kind_value;
1402
+ uint64_t infer_max_categories = 0;
1403
+ int has_numeric = 0;
1404
+ int has_categorical = 0;
1405
+ size_t source_len;
1406
+ memset(out, 0, sizeof(*out));
1407
+ const tf_transform_limits_v1 *limits = &runtime->limits;
1408
+ tf_transform_code code;
1409
+ if (!exact_keys(value, column_keys, 4))
1410
+ return tf_transform_set_error(
1411
+ error, TF_TRANSFORM_INVALID_RECIPE, "recipe column shape is invalid");
1412
+ kind = required_item(value, "kind");
1413
+ if (!exact_keys(kind, kind_keys, 4))
1414
+ return tf_transform_set_error(
1415
+ error, TF_TRANSFORM_INVALID_RECIPE,
1416
+ "prepared transform kind shape is invalid");
1417
+ kind_op = required_item(kind, "op");
1418
+ kind_value = required_item(kind, "value");
1419
+ if (json_string_equals(kind_op, "declared")) {
1420
+ if (!cJSON_IsNull(required_item(kind, "maxCategories"))
1421
+ || !cJSON_IsNull(required_item(kind, "rule")))
1422
+ return tf_transform_set_error(
1423
+ error, TF_TRANSFORM_INVALID_RECIPE,
1424
+ "declared transform kind is invalid");
1425
+ if (json_string_equals(kind_value, "numeric")) {
1426
+ out->kind = TF_TRANSFORM_KIND_NUMERIC;
1427
+ has_numeric = 1;
1428
+ } else if (json_string_equals(kind_value, "categorical")) {
1429
+ out->kind = TF_TRANSFORM_KIND_CATEGORICAL;
1430
+ has_categorical = 1;
1431
+ } else return tf_transform_set_error(
1432
+ error, TF_TRANSFORM_INVALID_RECIPE,
1433
+ "declared transform kind value is invalid");
1434
+ } else if (json_string_equals(kind_op, "infer")) {
1435
+ if (!cJSON_IsNull(kind_value)
1436
+ || !json_string_equals(
1437
+ required_item(kind, "rule"),
1438
+ "finite-integer-cardinality-v1")
1439
+ || !safe_uint64(
1440
+ required_item(kind, "maxCategories"), &infer_max_categories)
1441
+ || infer_max_categories < 2)
1442
+ return tf_transform_set_error(
1443
+ error, TF_TRANSFORM_INVALID_RECIPE,
1444
+ "inferred transform kind is invalid");
1445
+ if (infer_max_categories > limits->max_categories_per_column
1446
+ || infer_max_categories > limits->max_total_categories)
1447
+ return tf_transform_set_error(
1448
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
1449
+ "inferred category bound exceeds host limits");
1450
+ out->kind = TF_TRANSFORM_KIND_INFER;
1451
+ out->infer_max_categories = infer_max_categories;
1452
+ has_numeric = 1;
1453
+ has_categorical = 1;
1454
+ } else return tf_transform_set_error(
1455
+ error, TF_TRANSFORM_INVALID_RECIPE,
1456
+ "transform kind operation is unsupported");
1457
+ source_id = required_item(value, "sourceId");
1458
+ if (!cJSON_IsString(source_id) || !source_id->valuestring)
1459
+ return tf_transform_set_error(
1460
+ error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe sourceId");
1461
+ code = runtime_cstring_length(
1462
+ source_id->valuestring, (size_t)limits->max_string_bytes,
1463
+ runtime, &source_len, error);
1464
+ if (code != TF_TRANSFORM_OK || source_len == 0)
1465
+ return code != TF_TRANSFORM_OK ? code : tf_transform_set_error(
1466
+ error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe sourceId");
1467
+ out->source_id = (char *)tf_transform_resource_malloc(
1468
+ ledger, source_len + 1, error);
1469
+ if (!out->source_id) return ledger->last_code != TF_TRANSFORM_OK
1470
+ ? ledger->last_code : tf_transform_set_error(
1471
+ error, TF_TRANSFORM_ALLOCATION,
1472
+ "recipe sourceId allocation failed");
1473
+ code = tf_transform_copy_bytes_runtime(
1474
+ out->source_id, source_id->valuestring, source_len + 1,
1475
+ runtime, error);
1476
+ if (code != TF_TRANSFORM_OK) goto cleanup;
1477
+ out->source_id_len = source_len;
1478
+ numeric = required_item(value, "numeric");
1479
+ categorical = required_item(value, "categorical");
1480
+ if (has_numeric) {
1481
+ if (!parse_numeric_recipe_branch(numeric, out)) goto invalid;
1482
+ } else if (!cJSON_IsNull(numeric)) goto invalid;
1483
+ if (has_categorical) {
1484
+ if (!parse_categorical_recipe_branch(categorical, out)) goto invalid;
1485
+ const cJSON *categories = required_item(required_item(categorical, "encode"), "categories");
1486
+ if (cJSON_IsArray(categories)) {
1487
+ code = parse_fixed_categories(categories, runtime, ledger, out, error);
1488
+ if (code != TF_TRANSFORM_OK) goto cleanup;
1489
+ }
1490
+ } else if (!cJSON_IsNull(categorical)) goto invalid;
1491
+ return TF_TRANSFORM_OK;
1492
+ invalid:
1493
+ code = tf_transform_set_error(
1494
+ error, TF_TRANSFORM_INVALID_RECIPE,
1495
+ "invalid prepared-transform recipe column");
1496
+ cleanup:
1497
+ free(out->source_id);
1498
+ free(out->categorical_fixed);
1499
+ tf_transform_resource_release(ledger,
1500
+ (uint64_t)out->categorical_fixed_count * sizeof(*out->categorical_fixed));
1501
+ tf_transform_resource_release(ledger, (uint64_t)source_len + 1);
1502
+ memset(out, 0, sizeof(*out));
1503
+ return code;
1504
+ }
1505
+
1506
+ static tf_transform_code recipe_from_json_value(
1507
+ const cJSON *root, const tf_transform_runtime_copy *runtime,
1508
+ tf_transform_resource_ledger *ledger,
1509
+ tf_transform_recipe **out, tf_transform_error **error) {
1510
+ static const char *const top_keys[] = {
1511
+ "columns", "format", "outputDtype", "policyVersion",
1512
+ "semanticLimits", "version"
1513
+ };
1514
+ static const char *const semantic_keys[] = {
1515
+ "maxOutputColumns", "maxOutputElementsPerApply"
1516
+ };
1517
+ const tf_transform_limits_v1 *limits = &runtime->limits;
1518
+ const cJSON *columns;
1519
+ const cJSON *semantic;
1520
+ const cJSON *column_json;
1521
+ tf_transform_recipe *recipe = NULL;
1522
+ tf_transform_code code;
1523
+ uint64_t version;
1524
+ uint64_t policy_version;
1525
+ uint64_t max_columns;
1526
+ uint64_t max_elements;
1527
+ size_t count = 0;
1528
+ if (out) *out = NULL;
1529
+ if (!root || !runtime || !ledger || !out)
1530
+ return tf_transform_set_error(
1531
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
1532
+ "recipe value argument is null");
1533
+ if (!exact_keys(root, top_keys, 6)
1534
+ || !json_string_equals(required_item(root, "format"),
1535
+ "tranfi.transform-recipe")
1536
+ || !json_string_equals(required_item(root, "outputDtype"), "float64")
1537
+ || !safe_uint64(required_item(root, "version"), &version)
1538
+ || !safe_uint64(required_item(root, "policyVersion"), &policy_version))
1539
+ return tf_transform_set_error(
1540
+ error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe header");
1541
+ if (version != 1 || policy_version != 1)
1542
+ return tf_transform_set_error(
1543
+ error, TF_TRANSFORM_UNSUPPORTED_VERSION,
1544
+ "recipe or policy version is unsupported");
1545
+ semantic = required_item(root, "semanticLimits");
1546
+ if (!exact_keys(semantic, semantic_keys, 2)
1547
+ || !safe_uint64(required_item(semantic, "maxOutputColumns"), &max_columns)
1548
+ || !safe_uint64(required_item(semantic, "maxOutputElementsPerApply"),
1549
+ &max_elements)
1550
+ || max_columns == 0 || max_elements == 0)
1551
+ return tf_transform_set_error(
1552
+ error, TF_TRANSFORM_INVALID_RECIPE,
1553
+ "invalid recipe semantic limits");
1554
+ if (max_columns > limits->max_output_columns
1555
+ || max_elements > limits->max_output_elements_per_call)
1556
+ return tf_transform_set_error(
1557
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
1558
+ "recipe semantic limits exceed host limits");
1559
+ columns = required_item(root, "columns");
1560
+ if (!cJSON_IsArray(columns))
1561
+ return tf_transform_set_error(
1562
+ error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe columns");
1563
+ cJSON_ArrayForEach(column_json, columns) {
1564
+ if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
1565
+ code = tf_transform_poll_cancel(runtime, error);
1566
+ if (code != TF_TRANSFORM_OK) return code;
1567
+ }
1568
+ if (count == SIZE_MAX) return tf_transform_set_error(
1569
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
1570
+ "recipe column count overflows");
1571
+ ++count;
1572
+ }
1573
+ if (count == 0 || (uint64_t)count > limits->max_steps
1574
+ || (uint64_t)count > limits->max_input_columns)
1575
+ return tf_transform_set_error(
1576
+ error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe columns");
1577
+ if ((uint64_t)count > max_columns)
1578
+ return tf_transform_set_error(
1579
+ error, TF_TRANSFORM_INVALID_RECIPE,
1580
+ "recipe output columns exceed its semantic limit");
1581
+ if (count > SIZE_MAX / sizeof(tf_transform_recipe_column))
1582
+ return tf_transform_set_error(
1583
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
1584
+ "recipe column allocation overflows");
1585
+ recipe = (tf_transform_recipe *)tf_transform_resource_calloc(
1586
+ ledger, 1, sizeof(*recipe), error);
1587
+ if (!recipe) return ledger->last_code != TF_TRANSFORM_OK
1588
+ ? ledger->last_code : tf_transform_set_error(
1589
+ error, TF_TRANSFORM_ALLOCATION, "recipe allocation failed");
1590
+ atomic_init(&recipe->refcount, 1u);
1591
+ recipe->column_count = count;
1592
+ recipe->max_output_columns = max_columns;
1593
+ recipe->max_output_elements_per_apply = max_elements;
1594
+ recipe->columns = (tf_transform_recipe_column *)
1595
+ tf_transform_resource_calloc(
1596
+ ledger, count, sizeof(*recipe->columns), error);
1597
+ if (!recipe->columns) {
1598
+ code = ledger->last_code != TF_TRANSFORM_OK
1599
+ ? ledger->last_code : tf_transform_set_error(
1600
+ error, TF_TRANSFORM_ALLOCATION,
1601
+ "recipe column allocation failed");
1602
+ goto done;
1603
+ }
1604
+ column_json = columns->child;
1605
+ for (size_t i = 0; i < count; ++i, column_json = column_json->next) {
1606
+ code = tf_transform_poll_cancel(runtime, error);
1607
+ if (code != TF_TRANSFORM_OK) goto done;
1608
+ code = parse_recipe_column(
1609
+ column_json, runtime, ledger, &recipe->columns[i], error);
1610
+ if (code != TF_TRANSFORM_OK) goto done;
1611
+ for (size_t j = 0; j < i; ++j) {
1612
+ int equal = 0;
1613
+ code = runtime_span_equal(
1614
+ recipe->columns[i].source_id,
1615
+ recipe->columns[i].source_id_len,
1616
+ recipe->columns[j].source_id,
1617
+ recipe->columns[j].source_id_len,
1618
+ runtime, &equal, error);
1619
+ if (code != TF_TRANSFORM_OK) goto done;
1620
+ if (equal) {
1621
+ code = tf_transform_set_error(
1622
+ error, TF_TRANSFORM_INVALID_RECIPE,
1623
+ "recipe sourceIds must be unique");
1624
+ goto done;
1625
+ }
1626
+ }
1627
+ }
1628
+ *out = recipe;
1629
+ recipe = NULL;
1630
+ code = TF_TRANSFORM_OK;
1631
+ done:
1632
+ tf_transform_recipe_release(recipe);
1633
+ return code;
1634
+ }
1635
+
1636
+ tf_transform_code tf_transform_recipe_parse_numeric(
1637
+ const uint8_t *json, size_t json_len,
1638
+ const tf_transform_limits_v1 *limits,
1639
+ tf_transform_recipe **out, tf_transform_error **error) {
1640
+ cJSON *root = NULL;
1641
+ tf_transform_runtime_copy runtime;
1642
+ tf_transform_resource_ledger ledger;
1643
+ tf_transform_code code;
1644
+ uint64_t ast_resident = 0;
1645
+ if (out) *out = NULL;
1646
+ if (!json || json_len == 0 || !limits || !out)
1647
+ return tf_transform_set_error(
1648
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "recipe argument is null or empty");
1649
+ if (json_len > limits->max_recipe_bytes)
1650
+ return tf_transform_set_error(
1651
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "recipe exceeds byte limit");
1652
+ memset(&runtime, 0, sizeof(runtime));
1653
+ runtime.limits = *limits;
1654
+ tf_transform_resource_ledger_init(&ledger, &runtime);
1655
+ code = tf_transform_json_parse_bounded_runtime_ledger(
1656
+ json, json_len, &runtime, &ledger, &root,
1657
+ &ast_resident, NULL, error, TF_TRANSFORM_INVALID_RECIPE);
1658
+ if (code != TF_TRANSFORM_OK) return code;
1659
+ code = recipe_from_json_value(root, &runtime, &ledger, out, error);
1660
+ cJSON_Delete(root);
1661
+ tf_transform_resource_release(&ledger, ast_resident);
1662
+ return code;
1663
+ }
1664
+
1665
+ tf_transform_code tf_transform_recipe_from_json(
1666
+ const uint8_t *json, size_t json_len,
1667
+ const tf_transform_limits_v1 *limits,
1668
+ tf_transform_recipe **out, tf_transform_error **error) {
1669
+ tf_transform_limits_v1 copied;
1670
+ tf_transform_code code;
1671
+ if (out) *out = NULL;
1672
+ tf_transform_clear_error(error);
1673
+ if (!out) return tf_transform_set_error(
1674
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "recipe output is null");
1675
+ code = tf_transform_copy_limits(limits, &copied, error);
1676
+ if (code != TF_TRANSFORM_OK) return code;
1677
+ return tf_transform_recipe_parse_numeric(json, json_len, &copied, out, error);
1678
+ }
1679
+
1680
+ static cJSON *json_null(void) { return cJSON_CreateNull(); }
1681
+ static cJSON *json_string(const char *value) { return cJSON_CreateString(value); }
1682
+ static cJSON *json_number(double value) { return cJSON_CreateNumber(value); }
1683
+
1684
+ static int add_item(cJSON *object, const char *key, cJSON *item) {
1685
+ if (!item) return 0;
1686
+ if (!cJSON_AddItemToObject(object, key, item)) {
1687
+ cJSON_Delete(item);
1688
+ return 0;
1689
+ }
1690
+ return 1;
1691
+ }
1692
+
1693
+ static int transfer_item(cJSON *object, const char *key, cJSON **item) {
1694
+ cJSON *owned;
1695
+ if (!item) return 0;
1696
+ owned = *item;
1697
+ *item = NULL;
1698
+ return add_item(object, key, owned);
1699
+ }
1700
+
1701
+ static int transfer_array_item(cJSON *array, cJSON **item) {
1702
+ cJSON *owned;
1703
+ if (!item || !*item) return 0;
1704
+ owned = *item;
1705
+ *item = NULL;
1706
+ if (!cJSON_AddItemToArray(array, owned)) {
1707
+ cJSON_Delete(owned);
1708
+ return 0;
1709
+ }
1710
+ return 1;
1711
+ }
1712
+
1713
+ static cJSON *tagged_bits(uint64_t raw, uint32_t dtype) {
1714
+ static const char hex[] = "0123456789abcdef";
1715
+ cJSON *object = cJSON_CreateObject();
1716
+ char bits[17];
1717
+ size_t digits;
1718
+ if (!object) return NULL;
1719
+ if (dtype == TF_VIEW_FLOAT32) {
1720
+ raw &= UINT64_C(0xffffffff);
1721
+ digits = 8;
1722
+ if (!add_item(object, "t", json_string("f32"))) goto fail;
1723
+ } else if (dtype == TF_VIEW_FLOAT64) {
1724
+ digits = 16;
1725
+ if (!add_item(object, "t", json_string("f64"))) goto fail;
1726
+ } else goto fail;
1727
+ for (size_t i = 0; i < digits; ++i) {
1728
+ unsigned int shift = (unsigned int)((digits - 1 - i) * 4);
1729
+ bits[i] = hex[(raw >> shift) & 15u];
1730
+ }
1731
+ bits[digits] = '\0';
1732
+ if (!add_item(object, "v", json_string(bits))) goto fail;
1733
+ return object;
1734
+ fail:
1735
+ cJSON_Delete(object);
1736
+ return NULL;
1737
+ }
1738
+
1739
+ static cJSON *tagged_constant(double value, uint32_t dtype) {
1740
+ if (dtype == TF_VIEW_FLOAT32) {
1741
+ float narrowed = (float)value;
1742
+ uint32_t raw;
1743
+ memcpy(&raw, &narrowed, sizeof(raw));
1744
+ return tagged_bits(raw, dtype);
1745
+ }
1746
+ return tagged_bits(tf_transform_double_bits(value), dtype);
1747
+ }
1748
+
1749
+ typedef struct tf_json_build_budget {
1750
+ uint64_t nodes;
1751
+ uint64_t strings;
1752
+ uint64_t string_bytes;
1753
+ uint64_t object_keys;
1754
+ uint64_t objects;
1755
+ uint64_t max_string_bytes;
1756
+ uint64_t max_object_width;
1757
+ uint64_t depth;
1758
+ } tf_json_build_budget;
1759
+
1760
+ static int build_budget_add(uint64_t *target, uint64_t value) {
1761
+ if (*target > UINT64_MAX - value) return 0;
1762
+ *target += value;
1763
+ return 1;
1764
+ }
1765
+
1766
+ static int build_budget_node(tf_json_build_budget *budget) {
1767
+ return build_budget_add(&budget->nodes, 1);
1768
+ }
1769
+
1770
+ static int build_budget_string(
1771
+ tf_json_build_budget *budget, uint64_t bytes) {
1772
+ if (!build_budget_add(&budget->strings, 1)
1773
+ || !build_budget_add(&budget->string_bytes, bytes)) return 0;
1774
+ if (bytes > budget->max_string_bytes) budget->max_string_bytes = bytes;
1775
+ return 1;
1776
+ }
1777
+
1778
+ static int build_budget_key(
1779
+ tf_json_build_budget *budget, const char *key) {
1780
+ size_t len = strlen(key);
1781
+ return build_budget_add(&budget->object_keys, 1)
1782
+ && build_budget_string(budget, (uint64_t)len);
1783
+ }
1784
+
1785
+ static int build_budget_object(
1786
+ tf_json_build_budget *budget, uint64_t width) {
1787
+ if (!build_budget_node(budget)
1788
+ || !build_budget_add(&budget->objects, 1)) return 0;
1789
+ if (width > budget->max_object_width) budget->max_object_width = width;
1790
+ return 1;
1791
+ }
1792
+
1793
+ static int build_budget_array(tf_json_build_budget *budget) {
1794
+ return build_budget_node(budget);
1795
+ }
1796
+
1797
+ static int build_budget_primitive(
1798
+ tf_json_build_budget *budget, const char *key) {
1799
+ return build_budget_key(budget, key) && build_budget_node(budget);
1800
+ }
1801
+
1802
+ static int build_budget_string_property(
1803
+ tf_json_build_budget *budget, const char *key, uint64_t value_bytes) {
1804
+ return build_budget_primitive(budget, key)
1805
+ && build_budget_string(budget, value_bytes);
1806
+ }
1807
+
1808
+ static int build_budget_child(
1809
+ tf_json_build_budget *budget, const char *key) {
1810
+ return build_budget_key(budget, key);
1811
+ }
1812
+
1813
+ static int build_budget_tagged(
1814
+ tf_json_build_budget *budget, const char *key, uint64_t value_bytes) {
1815
+ return build_budget_child(budget, key)
1816
+ && build_budget_object(budget, 2)
1817
+ && build_budget_string_property(budget, "t", 3)
1818
+ && build_budget_string_property(budget, "v", value_bytes);
1819
+ }
1820
+
1821
+ static int build_budget_schema(
1822
+ const tf_transform_schema *schema, tf_json_build_budget *budget) {
1823
+ if (!schema || !build_budget_array(budget)) return 0;
1824
+ for (size_t i = 0; i < schema->field_count; ++i) {
1825
+ const tf_transform_schema_field_owned *field = &schema->fields[i];
1826
+ if (!build_budget_object(budget, schema->is_output ? 6 : 3)) return 0;
1827
+ if (schema->is_output) {
1828
+ if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_NONE) {
1829
+ if (!build_budget_primitive(budget, "category")) return 0;
1830
+ } else if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_OTHER) {
1831
+ if (!build_budget_child(budget, "category")
1832
+ || !build_budget_object(budget, 1)
1833
+ || !build_budget_string_property(budget, "t", 5)) return 0;
1834
+ } else {
1835
+ if (!build_budget_tagged(
1836
+ budget, "category",
1837
+ field->category_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
1838
+ }
1839
+ }
1840
+ if (!build_budget_string_property(budget, "dtype", 7)
1841
+ || !build_budget_string_property(
1842
+ budget, "id", (uint64_t)field->id_len)
1843
+ || !build_budget_string_property(
1844
+ budget, "name", (uint64_t)field->name_len)) return 0;
1845
+ if (schema->is_output
1846
+ && (!build_budget_string_property(
1847
+ budget, "role",
1848
+ field->role == TF_TRANSFORM_ROLE_ONEHOT ? 6 : 5)
1849
+ || !build_budget_string_property(
1850
+ budget, "sourceId", (uint64_t)field->source_id_len))) return 0;
1851
+ }
1852
+ if (budget->depth < 3) budget->depth = 3;
1853
+ return 1;
1854
+ }
1855
+
1856
+ static int build_budget_numeric_recipe_branch(
1857
+ const tf_transform_recipe_column *column, tf_json_build_budget *budget) {
1858
+ uint64_t impute_bytes = column->impute == TF_TRANSFORM_IMPUTE_CONSTANT ? 8
1859
+ : column->impute == TF_TRANSFORM_IMPUTE_MEDIAN ? 6 : 4;
1860
+ uint64_t normalize_bytes
1861
+ = column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD ? 8
1862
+ : column->normalize == TF_TRANSFORM_NORMALIZE_MINMAX ? 6 : 4;
1863
+ if (!build_budget_child(budget, "numeric")
1864
+ || !build_budget_object(budget, 2)
1865
+ || !build_budget_child(budget, "impute")
1866
+ || !build_budget_object(budget, 3)) return 0;
1867
+ if (column->impute == TF_TRANSFORM_IMPUTE_MEAN
1868
+ || column->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
1869
+ if (!build_budget_string_property(
1870
+ budget, "allMissing",
1871
+ column->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO ? 4 : 5))
1872
+ return 0;
1873
+ } else if (!build_budget_primitive(budget, "allMissing")) return 0;
1874
+ if (column->impute == TF_TRANSFORM_IMPUTE_CONSTANT) {
1875
+ if (!build_budget_tagged(
1876
+ budget, "constant",
1877
+ column->constant_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
1878
+ } else if (!build_budget_primitive(budget, "constant")) return 0;
1879
+ return build_budget_string_property(budget, "op", impute_bytes)
1880
+ && build_budget_child(budget, "normalize")
1881
+ && build_budget_object(budget, 2)
1882
+ && build_budget_primitive(budget, "ddof")
1883
+ && build_budget_string_property(budget, "op", normalize_bytes);
1884
+ }
1885
+
1886
+ static int build_budget_categorical_recipe_branch(
1887
+ const tf_transform_recipe_column *column, tf_json_build_budget *budget) {
1888
+ if (!build_budget_child(budget, "categorical")
1889
+ || !build_budget_object(budget, 2)
1890
+ || !build_budget_child(budget, "encode")
1891
+ || !build_budget_object(budget, 4)) return 0;
1892
+ if (column->categorical_fixed_count) {
1893
+ if (!build_budget_child(budget, "categories") || !build_budget_array(budget)) return 0;
1894
+ for (size_t i = 0; i < column->categorical_fixed_count; ++i)
1895
+ if (!build_budget_object(budget, 2)
1896
+ || !build_budget_string_property(budget, "t", 3)
1897
+ || !build_budget_string_property(budget, "v",
1898
+ column->categorical_fixed_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
1899
+ if (budget->depth < 9) budget->depth = 9;
1900
+ } else if (column->categorical_discover
1901
+ ? !build_budget_string_property(budget, "categories", 8)
1902
+ : !build_budget_primitive(budget, "categories")) return 0;
1903
+ if (!build_budget_string_property(
1904
+ budget, "op",
1905
+ column->categorical_encode == TF_TRANSFORM_ENCODE_LABEL ? 5
1906
+ : column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT ? 6 : 4)
1907
+ || !build_budget_primitive(budget, "sentinelLabel")) return 0;
1908
+ if (column->categorical_unknown == TF_TRANSFORM_UNKNOWN_NONE) {
1909
+ if (!build_budget_primitive(budget, "unknown")) return 0;
1910
+ } else if (!build_budget_string_property(
1911
+ budget, "unknown",
1912
+ column->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? 8
1913
+ : column->categorical_unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO ? 8 : 5))
1914
+ return 0;
1915
+ if (!build_budget_child(budget, "impute")
1916
+ || !build_budget_object(budget, 3)) return 0;
1917
+ if (column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
1918
+ if (!build_budget_primitive(budget, "allMissing")) return 0;
1919
+ } else if (!build_budget_string_property(
1920
+ budget, "allMissing",
1921
+ column->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO ? 4 : 5))
1922
+ return 0;
1923
+ return build_budget_primitive(budget, "constant")
1924
+ && build_budget_string_property(budget, "op", 4);
1925
+ }
1926
+
1927
+ static int build_budget_recipe(
1928
+ const tf_transform_recipe *recipe, tf_json_build_budget *budget) {
1929
+ if (!recipe || !build_budget_object(budget, 6)
1930
+ || !build_budget_child(budget, "columns")
1931
+ || !build_budget_array(budget)) return 0;
1932
+ for (size_t i = 0; i < recipe->column_count; ++i) {
1933
+ const tf_transform_recipe_column *column = &recipe->columns[i];
1934
+ if (!build_budget_object(budget, 4)
1935
+ || !build_budget_child(budget, "kind")
1936
+ || !build_budget_object(budget, 4)
1937
+ || (column->kind == TF_TRANSFORM_KIND_INFER
1938
+ ? (!build_budget_primitive(budget, "maxCategories")
1939
+ || !build_budget_string_property(budget, "op", 5)
1940
+ || !build_budget_string_property(budget, "rule", 29)
1941
+ || !build_budget_primitive(budget, "value"))
1942
+ : (!build_budget_primitive(budget, "maxCategories")
1943
+ || !build_budget_string_property(budget, "op", 8)
1944
+ || !build_budget_primitive(budget, "rule")
1945
+ || !build_budget_string_property(
1946
+ budget, "value",
1947
+ column->kind == TF_TRANSFORM_KIND_CATEGORICAL ? 11 : 7))))
1948
+ return 0;
1949
+ if (column->kind == TF_TRANSFORM_KIND_NUMERIC) {
1950
+ if (!build_budget_primitive(budget, "categorical")
1951
+ || !build_budget_numeric_recipe_branch(column, budget)) return 0;
1952
+ } else if (column->kind == TF_TRANSFORM_KIND_CATEGORICAL) {
1953
+ if (!build_budget_categorical_recipe_branch(column, budget)
1954
+ || !build_budget_primitive(budget, "numeric")) return 0;
1955
+ } else if (column->kind == TF_TRANSFORM_KIND_INFER) {
1956
+ if (!build_budget_categorical_recipe_branch(column, budget)
1957
+ || !build_budget_numeric_recipe_branch(column, budget)) return 0;
1958
+ } else return 0;
1959
+ if (!build_budget_string_property(
1960
+ budget, "sourceId", (uint64_t)column->source_id_len)) return 0;
1961
+ }
1962
+ if (!build_budget_string_property(budget, "format", 23)
1963
+ || !build_budget_string_property(budget, "outputDtype", 7)
1964
+ || !build_budget_primitive(budget, "policyVersion")
1965
+ || !build_budget_child(budget, "semanticLimits")
1966
+ || !build_budget_object(budget, 2)
1967
+ || !build_budget_primitive(budget, "maxOutputColumns")
1968
+ || !build_budget_primitive(budget, "maxOutputElementsPerApply")
1969
+ || !build_budget_primitive(budget, "version")) return 0;
1970
+ if (budget->depth < 8) budget->depth = 8;
1971
+ return 1;
1972
+ }
1973
+
1974
+ static int build_budget_step(
1975
+ const tf_transform_recipe_column *recipe,
1976
+ const tf_transform_column_state *column_state,
1977
+ tf_json_build_budget *budget) {
1978
+ if (column_state->kind == TF_TRANSFORM_KIND_CATEGORICAL) {
1979
+ const tf_transform_categorical_state *state
1980
+ = &column_state->value.categorical;
1981
+ uint64_t digits = state->source_dtype == TF_VIEW_FLOAT32 ? 8 : 16;
1982
+ if (column_state->kind != TF_TRANSFORM_KIND_CATEGORICAL
1983
+ || !build_budget_object(budget, 6)
1984
+ || !build_budget_child(budget, "categorical")
1985
+ || !build_budget_object(budget, 3)
1986
+ || !build_budget_child(budget, "categories")
1987
+ || !build_budget_array(budget)) return 0;
1988
+ for (size_t i = 0; i < state->category_count; ++i)
1989
+ if (!build_budget_object(budget, 2)
1990
+ || !build_budget_string_property(budget, "t", 3)
1991
+ || !build_budget_string_property(budget, "v", digits)) return 0;
1992
+ if (!build_budget_child(budget, "encode")
1993
+ || !build_budget_object(budget, 4)
1994
+ || !build_budget_string_property(
1995
+ budget, "op",
1996
+ state->encode == TF_TRANSFORM_ENCODE_LABEL ? 5
1997
+ : state->encode == TF_TRANSFORM_ENCODE_ONEHOT ? 6 : 4)
1998
+ || !build_budget_primitive(budget, "otherOrdinal")
1999
+ || !build_budget_primitive(budget, "sentinelLabel")) return 0;
2000
+ if (state->unknown == TF_TRANSFORM_UNKNOWN_NONE) {
2001
+ if (!build_budget_primitive(budget, "unknown")) return 0;
2002
+ } else if (!build_budget_string_property(
2003
+ budget, "unknown",
2004
+ state->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? 8
2005
+ : state->unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO ? 8 : 5))
2006
+ return 0;
2007
+ if (!build_budget_child(budget, "impute")
2008
+ || !build_budget_object(budget, 3)) return 0;
2009
+ if (state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
2010
+ if (!build_budget_primitive(budget, "allMissing")
2011
+ || !build_budget_string_property(budget, "op", 4)
2012
+ || !build_budget_primitive(budget, "value")) return 0;
2013
+ } else if (!build_budget_string_property(budget, "allMissing", 5)
2014
+ || !build_budget_string_property(budget, "op", 4)
2015
+ || !build_budget_tagged(budget, "value", digits)) return 0;
2016
+ if (!build_budget_string_property(budget, "kind", 11)
2017
+ || !build_budget_primitive(budget, "numeric")
2018
+ || !build_budget_string_property(
2019
+ budget, "sourceId", (uint64_t)recipe->source_id_len)
2020
+ || !build_budget_primitive(budget, "stateVersion")) return 0;
2021
+ return 1;
2022
+ }
2023
+ {
2024
+ const tf_transform_numeric_state *state = &column_state->value.numeric;
2025
+ if (column_state->kind != TF_TRANSFORM_KIND_NUMERIC) return 0;
2026
+ if (!build_budget_object(budget, 6)
2027
+ || !build_budget_primitive(budget, "categorical")
2028
+ || !build_budget_string_property(budget, "kind", 7)
2029
+ || !build_budget_child(budget, "numeric")
2030
+ || !build_budget_object(budget, 2)
2031
+ || !build_budget_child(budget, "impute")
2032
+ || !build_budget_object(budget, 3)) return 0;
2033
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_MEAN
2034
+ || recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
2035
+ if (!build_budget_string_property(
2036
+ budget, "allMissing", 5)) return 0;
2037
+ } else if (!build_budget_primitive(budget, "allMissing")) return 0;
2038
+ if (!build_budget_string_property(budget, "op", 8)) return 0;
2039
+ if (state->has_impute_value) {
2040
+ if (!build_budget_tagged(budget, "value", 16)) return 0;
2041
+ } else if (!build_budget_primitive(budget, "value")) return 0;
2042
+ if (!build_budget_child(budget, "normalize")
2043
+ || !build_budget_object(budget, 5)
2044
+ || !build_budget_primitive(budget, "ddof")
2045
+ || !build_budget_tagged(budget, "location", 16)
2046
+ || !build_budget_string_property(budget, "op", 8)
2047
+ || !build_budget_tagged(budget, "scale", 16)
2048
+ || !build_budget_string_property(
2049
+ budget, "sourceId", (uint64_t)recipe->source_id_len)
2050
+ || !build_budget_primitive(budget, "stateVersion")) return 0;
2051
+ return 1;
2052
+ }
2053
+ }
2054
+
2055
+ static tf_transform_code check_build_budget(
2056
+ const tf_json_build_budget *budget,
2057
+ const tf_transform_limits_v1 *limits,
2058
+ uint64_t extra_allocations, uint64_t output_copies,
2059
+ tf_transform_error **error) {
2060
+ uint64_t allocations = budget->nodes;
2061
+ uint64_t resident;
2062
+ uint64_t output_upper;
2063
+ uint64_t sort_bytes;
2064
+ if (!build_budget_add(&allocations, budget->strings)
2065
+ || !build_budget_add(&allocations, budget->objects)
2066
+ || !build_budget_add(&allocations, extra_allocations)) goto overflow;
2067
+ if (budget->nodes > UINT64_MAX / sizeof(cJSON)) goto overflow;
2068
+ resident = budget->nodes * sizeof(cJSON);
2069
+ if (!build_budget_add(&resident, budget->string_bytes)
2070
+ || !build_budget_add(&resident, budget->strings)) goto overflow;
2071
+ if (budget->max_object_width > UINT64_MAX / sizeof(cJSON *)) goto overflow;
2072
+ sort_bytes = budget->max_object_width * sizeof(cJSON *);
2073
+ if (!build_budget_add(&resident, sort_bytes)) goto overflow;
2074
+ if (budget->nodes > UINT64_MAX / 32) goto overflow;
2075
+ output_upper = budget->nodes * 32;
2076
+ if (budget->string_bytes > (UINT64_MAX - output_upper) / 6)
2077
+ goto overflow;
2078
+ output_upper += budget->string_bytes * 6;
2079
+ if (output_upper > limits->max_plan_bytes)
2080
+ output_upper = limits->max_plan_bytes;
2081
+ if (output_copies != 0 && output_upper > UINT64_MAX / output_copies)
2082
+ goto overflow;
2083
+ if (!build_budget_add(&resident, output_upper * output_copies)) goto overflow;
2084
+ if (budget->depth > limits->max_json_depth
2085
+ || budget->object_keys > limits->max_object_keys
2086
+ || budget->string_bytes > limits->max_decoded_string_bytes
2087
+ || budget->max_string_bytes > limits->max_string_bytes
2088
+ || allocations > limits->max_allocations_per_session
2089
+ || resident > limits->max_resident_state_bytes
2090
+ || sizeof(cJSON) > limits->max_allocation_bytes
2091
+ || sort_bytes > limits->max_allocation_bytes
2092
+ || (budget->strings != 0
2093
+ && (budget->max_string_bytes == UINT64_MAX
2094
+ || budget->max_string_bytes + 1
2095
+ > limits->max_allocation_bytes)))
2096
+ return tf_transform_set_error(
2097
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2098
+ "JSON construction exceeds configured limits");
2099
+ return TF_TRANSFORM_OK;
2100
+ overflow:
2101
+ return tf_transform_set_error(
2102
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2103
+ "JSON construction resource estimate overflows");
2104
+ }
2105
+
2106
+ tf_transform_code tf_transform_schema_json_preflight(
2107
+ const tf_transform_schema *schema, const tf_transform_limits_v1 *limits,
2108
+ tf_transform_error **error) {
2109
+ tf_json_build_budget budget = {0};
2110
+ if (!schema || !limits)
2111
+ return tf_transform_set_error(
2112
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2113
+ "schema JSON preflight argument is null");
2114
+ if ((uint64_t)schema->field_count
2115
+ > (schema->is_output ? limits->max_output_columns
2116
+ : limits->max_input_columns)
2117
+ || !build_budget_schema(schema, &budget))
2118
+ return tf_transform_set_error(
2119
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2120
+ "schema JSON construction estimate overflows");
2121
+ return check_build_budget(&budget, limits, 1, 1, error);
2122
+ }
2123
+
2124
+ tf_transform_code tf_transform_plan_json_preflight(
2125
+ const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
2126
+ tf_transform_error **error) {
2127
+ tf_json_build_budget fingerprint = {0};
2128
+ tf_json_build_budget plan_json = {0};
2129
+ tf_transform_code code;
2130
+ uint64_t total_allocations;
2131
+ uint64_t total_categories = 0;
2132
+ if (!plan || !limits)
2133
+ return tf_transform_set_error(
2134
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2135
+ "plan JSON preflight argument is null");
2136
+ if ((uint64_t)plan->input_schema.field_count > limits->max_input_columns
2137
+ || (uint64_t)plan->output_schema.field_count > limits->max_output_columns
2138
+ || (uint64_t)plan->input_schema.field_count > limits->max_steps)
2139
+ return tf_transform_set_error(
2140
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2141
+ "plan JSON structure exceeds configured limits");
2142
+ for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
2143
+ uint64_t count;
2144
+ if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
2145
+ && (plan->recipe->columns[i].infer_max_categories
2146
+ > limits->max_categories_per_column
2147
+ || plan->recipe->columns[i].infer_max_categories
2148
+ > limits->max_total_categories))
2149
+ return tf_transform_set_error(
2150
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2151
+ "inferred category bound exceeds configured limits");
2152
+ if (plan->states[i].kind != TF_TRANSFORM_KIND_CATEGORICAL) continue;
2153
+ count = (uint64_t)plan->states[i].value.categorical.category_count;
2154
+ if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
2155
+ && (count < 2
2156
+ || count > plan->recipe->columns[i].infer_max_categories))
2157
+ return tf_transform_set_error(
2158
+ error, TF_TRANSFORM_INTERNAL,
2159
+ "inferred categorical plan state is inconsistent");
2160
+ if (count > limits->max_categories_per_column
2161
+ || total_categories > UINT64_MAX - count
2162
+ || total_categories + count > limits->max_total_categories)
2163
+ return tf_transform_set_error(
2164
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2165
+ "plan categories exceed configured limits");
2166
+ total_categories += count;
2167
+ }
2168
+ if (!build_budget_object(&fingerprint, 3)
2169
+ || !build_budget_child(&fingerprint, "inputSchema")
2170
+ || !build_budget_schema(&plan->input_schema, &fingerprint)
2171
+ || !build_budget_primitive(&fingerprint, "policyVersion")
2172
+ || !build_budget_child(&fingerprint, "recipe")
2173
+ || !build_budget_recipe(plan->recipe, &fingerprint)) goto overflow;
2174
+ if (!build_budget_object(&plan_json, 7)
2175
+ || !build_budget_string_property(&plan_json, "format", 21)
2176
+ || !build_budget_child(&plan_json, "inputSchema")
2177
+ || !build_budget_schema(&plan->input_schema, &plan_json)
2178
+ || !build_budget_child(&plan_json, "outputSchema")
2179
+ || !build_budget_schema(&plan->output_schema, &plan_json)
2180
+ || !build_budget_child(&plan_json, "recipe")
2181
+ || !build_budget_recipe(plan->recipe, &plan_json)
2182
+ || !build_budget_string_property(&plan_json, "recipeSha256", 64)
2183
+ || !build_budget_child(&plan_json, "steps")
2184
+ || !build_budget_array(&plan_json)) goto overflow;
2185
+ for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
2186
+ if (!build_budget_step(
2187
+ &plan->recipe->columns[i], &plan->states[i], &plan_json))
2188
+ goto overflow;
2189
+ }
2190
+ if (!build_budget_primitive(&plan_json, "version")) goto overflow;
2191
+ if (plan_json.depth < 8) plan_json.depth = 8;
2192
+ code = check_build_budget(&fingerprint, limits, 1, 1, error);
2193
+ if (code != TF_TRANSFORM_OK) return code;
2194
+ code = check_build_budget(&plan_json, limits, 2, 2, error);
2195
+ if (code != TF_TRANSFORM_OK) return code;
2196
+ total_allocations = fingerprint.nodes;
2197
+ if (!build_budget_add(&total_allocations, fingerprint.strings)
2198
+ || !build_budget_add(&total_allocations, fingerprint.objects)
2199
+ || !build_budget_add(&total_allocations, plan_json.nodes)
2200
+ || !build_budget_add(&total_allocations, plan_json.strings)
2201
+ || !build_budget_add(&total_allocations, plan_json.objects)
2202
+ || !build_budget_add(&total_allocations, 3)) goto overflow;
2203
+ if (total_allocations > limits->max_allocations_per_session)
2204
+ return tf_transform_set_error(
2205
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2206
+ "plan JSON allocation count exceeds configured limit");
2207
+ return TF_TRANSFORM_OK;
2208
+ overflow:
2209
+ return tf_transform_set_error(
2210
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2211
+ "plan JSON construction estimate overflows");
2212
+ }
2213
+
2214
+ static cJSON *recipe_numeric_branch_to_json(
2215
+ const tf_transform_recipe_column *column) {
2216
+ cJSON *numeric = cJSON_CreateObject();
2217
+ cJSON *impute = cJSON_CreateObject();
2218
+ cJSON *normalize = cJSON_CreateObject();
2219
+ const char *impute_name;
2220
+ const char *normalize_name;
2221
+ if (!numeric || !impute || !normalize) goto fail;
2222
+ if (column->impute == TF_TRANSFORM_IMPUTE_MEAN
2223
+ || column->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
2224
+ if (!add_item(impute, "allMissing", json_string(
2225
+ column->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
2226
+ ? "zero" : "error"))) goto fail;
2227
+ } else if (!add_item(impute, "allMissing", json_null())) goto fail;
2228
+ if (column->impute == TF_TRANSFORM_IMPUTE_CONSTANT) {
2229
+ if (!add_item(impute, "constant", tagged_constant(
2230
+ column->constant, column->constant_dtype))) goto fail;
2231
+ } else if (!add_item(impute, "constant", json_null())) goto fail;
2232
+ impute_name = column->impute == TF_TRANSFORM_IMPUTE_NONE ? "none"
2233
+ : column->impute == TF_TRANSFORM_IMPUTE_ZERO ? "zero"
2234
+ : column->impute == TF_TRANSFORM_IMPUTE_CONSTANT ? "constant"
2235
+ : column->impute == TF_TRANSFORM_IMPUTE_MEAN ? "mean" : "median";
2236
+ if (!add_item(impute, "op", json_string(impute_name))
2237
+ || !transfer_item(numeric, "impute", &impute)) goto fail;
2238
+ if (column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
2239
+ if (!add_item(normalize, "ddof", json_number(column->ddof))) goto fail;
2240
+ } else if (!add_item(normalize, "ddof", json_null())) goto fail;
2241
+ normalize_name = column->normalize == TF_TRANSFORM_NORMALIZE_NONE ? "none"
2242
+ : column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD
2243
+ ? "standard" : "minmax";
2244
+ if (!add_item(normalize, "op", json_string(normalize_name))
2245
+ || !transfer_item(numeric, "normalize", &normalize)) goto fail;
2246
+ return numeric;
2247
+ fail:
2248
+ cJSON_Delete(numeric);
2249
+ cJSON_Delete(impute);
2250
+ cJSON_Delete(normalize);
2251
+ return NULL;
2252
+ }
2253
+
2254
+ static cJSON *recipe_categories_to_json(const tf_transform_recipe_column *column) {
2255
+ if (!column->categorical_fixed_count)
2256
+ return column->categorical_discover ? json_string("discover") : json_null();
2257
+ cJSON *categories = cJSON_CreateArray();
2258
+ if (!categories) return NULL;
2259
+ for (size_t i = 0; i < column->categorical_fixed_count; ++i) {
2260
+ cJSON *value = tagged_bits(column->categorical_fixed[i].bits,
2261
+ column->categorical_fixed_dtype);
2262
+ if (!value || !transfer_array_item(categories, &value)) {
2263
+ cJSON_Delete(value);
2264
+ cJSON_Delete(categories);
2265
+ return NULL;
2266
+ }
2267
+ }
2268
+ return categories;
2269
+ }
2270
+
2271
+ static cJSON *recipe_categorical_branch_to_json(
2272
+ const tf_transform_recipe_column *column) {
2273
+ cJSON *categorical = cJSON_CreateObject();
2274
+ cJSON *impute = cJSON_CreateObject();
2275
+ cJSON *encode = cJSON_CreateObject();
2276
+ cJSON *sentinel = column->categorical_has_sentinel_label
2277
+ ? json_number((double)column->categorical_sentinel_label) : json_null();
2278
+ cJSON *unknown = column->categorical_unknown == TF_TRANSFORM_UNKNOWN_NONE
2279
+ ? json_null() : json_string(
2280
+ column->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL
2281
+ ? "sentinel"
2282
+ : column->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER
2283
+ ? "other"
2284
+ : column->categorical_unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO
2285
+ ? "all_zero" : "error");
2286
+ if (!categorical || !impute || !encode || !sentinel || !unknown
2287
+ || !add_item(encode, "categories", recipe_categories_to_json(column))
2288
+ || !add_item(encode, "op", json_string(
2289
+ column->categorical_encode == TF_TRANSFORM_ENCODE_LABEL ? "label"
2290
+ : column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT
2291
+ ? "onehot" : "none"))
2292
+ || !transfer_item(encode, "sentinelLabel", &sentinel)
2293
+ || !transfer_item(encode, "unknown", &unknown)
2294
+ || !transfer_item(categorical, "encode", &encode)
2295
+ || !add_item(impute, "allMissing",
2296
+ column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
2297
+ ? json_null() : json_string(
2298
+ column->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
2299
+ ? "zero" : "error"))
2300
+ || !add_item(impute, "constant", json_null())
2301
+ || !add_item(impute, "op", json_string(
2302
+ column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
2303
+ ? "none" : "mode"))
2304
+ || !transfer_item(categorical, "impute", &impute)) goto fail;
2305
+ return categorical;
2306
+ fail:
2307
+ cJSON_Delete(categorical);
2308
+ cJSON_Delete(impute);
2309
+ cJSON_Delete(encode);
2310
+ cJSON_Delete(sentinel);
2311
+ cJSON_Delete(unknown);
2312
+ return NULL;
2313
+ }
2314
+
2315
+ cJSON *tf_transform_recipe_to_json(const tf_transform_recipe *recipe) {
2316
+ cJSON *root = cJSON_CreateObject();
2317
+ cJSON *columns = cJSON_CreateArray();
2318
+ cJSON *semantic = NULL;
2319
+ if (!recipe || !root || !columns) goto fail;
2320
+ if (!transfer_item(root, "columns", &columns)) goto fail;
2321
+ for (size_t i = 0; i < recipe->column_count; ++i) {
2322
+ const tf_transform_recipe_column *column = &recipe->columns[i];
2323
+ cJSON *entry = cJSON_CreateObject();
2324
+ cJSON *kind = cJSON_CreateObject();
2325
+ cJSON *numeric = column->kind == TF_TRANSFORM_KIND_CATEGORICAL
2326
+ ? json_null() : recipe_numeric_branch_to_json(column);
2327
+ cJSON *categorical = column->kind == TF_TRANSFORM_KIND_NUMERIC
2328
+ ? json_null() : recipe_categorical_branch_to_json(column);
2329
+ cJSON *max_categories = column->kind == TF_TRANSFORM_KIND_INFER
2330
+ ? json_number((double)column->infer_max_categories) : json_null();
2331
+ cJSON *rule = column->kind == TF_TRANSFORM_KIND_INFER
2332
+ ? json_string("finite-integer-cardinality-v1") : json_null();
2333
+ cJSON *kind_value = column->kind == TF_TRANSFORM_KIND_INFER
2334
+ ? json_null() : json_string(
2335
+ column->kind == TF_TRANSFORM_KIND_CATEGORICAL
2336
+ ? "categorical" : "numeric");
2337
+ if (!entry || !kind || !numeric || !categorical || !max_categories
2338
+ || !rule || !kind_value
2339
+ || !transfer_item(entry, "categorical", &categorical)
2340
+ || !transfer_item(kind, "maxCategories", &max_categories)
2341
+ || !add_item(kind, "op", json_string(
2342
+ column->kind == TF_TRANSFORM_KIND_INFER ? "infer" : "declared"))
2343
+ || !transfer_item(kind, "rule", &rule)
2344
+ || !transfer_item(kind, "value", &kind_value)
2345
+ || !transfer_item(entry, "kind", &kind)
2346
+ || !transfer_item(entry, "numeric", &numeric)
2347
+ || !add_item(entry, "sourceId", json_string(column->source_id))
2348
+ || !transfer_array_item(
2349
+ cJSON_GetObjectItemCaseSensitive(root, "columns"), &entry)) {
2350
+ cJSON_Delete(entry);
2351
+ cJSON_Delete(kind);
2352
+ cJSON_Delete(numeric);
2353
+ cJSON_Delete(categorical);
2354
+ cJSON_Delete(max_categories);
2355
+ cJSON_Delete(rule);
2356
+ cJSON_Delete(kind_value);
2357
+ goto fail;
2358
+ }
2359
+ continue;
2360
+ }
2361
+ if (!add_item(root, "format", json_string("tranfi.transform-recipe"))
2362
+ || !add_item(root, "outputDtype", json_string("float64"))
2363
+ || !add_item(root, "policyVersion", json_number(1))) goto fail;
2364
+ semantic = cJSON_CreateObject();
2365
+ if (!semantic
2366
+ || !add_item(semantic, "maxOutputColumns",
2367
+ json_number((double)recipe->max_output_columns))
2368
+ || !add_item(semantic, "maxOutputElementsPerApply",
2369
+ json_number((double)recipe->max_output_elements_per_apply)))
2370
+ goto fail;
2371
+ if (!transfer_item(root, "semanticLimits", &semantic)) goto fail;
2372
+ if (!add_item(root, "version", json_number(1))) goto fail;
2373
+ return root;
2374
+ fail:
2375
+ cJSON_Delete(columns);
2376
+ cJSON_Delete(semantic);
2377
+ cJSON_Delete(root);
2378
+ return NULL;
2379
+ }
2380
+
2381
+ cJSON *tf_transform_schema_to_json(const tf_transform_schema *schema) {
2382
+ cJSON *array = cJSON_CreateArray();
2383
+ if (!schema || !array) return NULL;
2384
+ for (size_t i = 0; i < schema->field_count; ++i) {
2385
+ const tf_transform_schema_field_owned *field = &schema->fields[i];
2386
+ cJSON *entry = cJSON_CreateObject();
2387
+ if (!entry) goto fail;
2388
+ if (schema->is_output) {
2389
+ cJSON *category = NULL;
2390
+ if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_NONE)
2391
+ category = json_null();
2392
+ else if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_OTHER) {
2393
+ category = cJSON_CreateObject();
2394
+ if (category && !add_item(category, "t", json_string("other"))) {
2395
+ cJSON_Delete(category);
2396
+ category = NULL;
2397
+ }
2398
+ } else category = tagged_bits(
2399
+ field->category_bits, field->category_dtype);
2400
+ if (!category || !transfer_item(entry, "category", &category)) {
2401
+ cJSON_Delete(category);
2402
+ goto entry_fail;
2403
+ }
2404
+ }
2405
+ if (!add_item(entry, "dtype", json_string(
2406
+ field->dtype == TF_VIEW_FLOAT32 ? "float32" : "float64"))
2407
+ || !add_item(entry, "id", json_string(field->id))
2408
+ || !add_item(entry, "name", json_string(field->name))) goto entry_fail;
2409
+ if (schema->is_output
2410
+ && (!add_item(entry, "role", json_string(
2411
+ field->role == TF_TRANSFORM_ROLE_LABEL ? "label"
2412
+ : field->role == TF_TRANSFORM_ROLE_ONEHOT ? "onehot" : "value"))
2413
+ || !add_item(entry, "sourceId", json_string(field->source_id))))
2414
+ goto entry_fail;
2415
+ if (!transfer_array_item(array, &entry)) goto entry_fail;
2416
+ continue;
2417
+ entry_fail:
2418
+ cJSON_Delete(entry);
2419
+ goto fail;
2420
+ }
2421
+ return array;
2422
+ fail:
2423
+ cJSON_Delete(array);
2424
+ return NULL;
2425
+ }
2426
+
2427
+ static void digest_hex(const uint8_t digest[32], char out[65]) {
2428
+ static const char digits[] = "0123456789abcdef";
2429
+ for (size_t i = 0; i < 32; ++i) {
2430
+ out[i * 2] = digits[digest[i] >> 4];
2431
+ out[i * 2 + 1] = digits[digest[i] & 15u];
2432
+ }
2433
+ out[64] = '\0';
2434
+ }
2435
+
2436
+ tf_transform_code tf_transform_plan_recipe_fingerprint(
2437
+ const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
2438
+ char out[65], tf_transform_error **error) {
2439
+ cJSON *object = NULL;
2440
+ cJSON *schema = NULL;
2441
+ cJSON *recipe_json = NULL;
2442
+ uint8_t *canonical = NULL;
2443
+ size_t canonical_len = 0;
2444
+ uint8_t digest[32];
2445
+ tf_transform_code code;
2446
+ if (!plan || !limits || !out) return tf_transform_set_error(
2447
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2448
+ "recipe fingerprint argument is null");
2449
+ object = cJSON_CreateObject();
2450
+ if (!object) return tf_transform_set_error(
2451
+ error, TF_TRANSFORM_ALLOCATION, "recipe fingerprint allocation failed");
2452
+ schema = tf_transform_schema_to_json(&plan->input_schema);
2453
+ recipe_json = tf_transform_recipe_to_json(plan->recipe);
2454
+ if (!schema || !recipe_json
2455
+ || !transfer_item(object, "inputSchema", &schema)) goto allocation_failed;
2456
+ if (!add_item(object, "policyVersion", json_number(1))
2457
+ || !transfer_item(object, "recipe", &recipe_json)) goto allocation_failed;
2458
+ code = tf_transform_json_print_canonical(
2459
+ object, limits, &canonical, &canonical_len, error);
2460
+ if (code != TF_TRANSFORM_OK) goto done;
2461
+ tf_transform_sha256(canonical, canonical_len, digest);
2462
+ digest_hex(digest, out);
2463
+ done:
2464
+ tf_transform_bytes_free(&canonical, &canonical_len);
2465
+ cJSON_Delete(schema);
2466
+ cJSON_Delete(recipe_json);
2467
+ cJSON_Delete(object);
2468
+ return code;
2469
+ allocation_failed:
2470
+ code = tf_transform_set_error(
2471
+ error, TF_TRANSFORM_ALLOCATION, "recipe fingerprint JSON failed");
2472
+ goto done;
2473
+ }
2474
+
2475
+ static tf_transform_code imported_recipe_fingerprint(
2476
+ const cJSON *plan_root, const tf_transform_runtime_copy *runtime,
2477
+ tf_transform_resource_ledger *ledger, char out[65],
2478
+ tf_transform_error **error) {
2479
+ const cJSON *input_schema = required_item(plan_root, "inputSchema");
2480
+ const cJSON *recipe = required_item(plan_root, "recipe");
2481
+ cJSON object;
2482
+ cJSON input_copy;
2483
+ cJSON policy;
2484
+ cJSON recipe_copy;
2485
+ uint8_t *canonical = NULL;
2486
+ size_t canonical_len = 0;
2487
+ uint8_t digest[32];
2488
+ tf_transform_code code;
2489
+ if (!input_schema || !recipe)
2490
+ return tf_transform_set_error(
2491
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2492
+ "recipe fingerprint inputs are missing");
2493
+ memset(&object, 0, sizeof(object));
2494
+ input_copy = *input_schema;
2495
+ memset(&policy, 0, sizeof(policy));
2496
+ recipe_copy = *recipe;
2497
+ object.type = cJSON_Object;
2498
+ object.child = &input_copy;
2499
+ input_copy.string = (char *)"inputSchema";
2500
+ input_copy.prev = NULL;
2501
+ input_copy.next = &policy;
2502
+ policy.type = cJSON_Number;
2503
+ policy.valueint = 1;
2504
+ policy.valuedouble = 1.0;
2505
+ policy.string = (char *)"policyVersion";
2506
+ policy.prev = &input_copy;
2507
+ policy.next = &recipe_copy;
2508
+ recipe_copy.string = (char *)"recipe";
2509
+ recipe_copy.prev = &policy;
2510
+ recipe_copy.next = NULL;
2511
+ code = tf_transform_json_print_canonical_runtime_ledger(
2512
+ &object, runtime, ledger, &canonical, &canonical_len, error);
2513
+ if (code != TF_TRANSFORM_OK) return code;
2514
+ code = tf_transform_sha256_runtime(
2515
+ canonical, canonical_len, runtime, digest, error);
2516
+ if (code == TF_TRANSFORM_OK) digest_hex(digest, out);
2517
+ free(canonical);
2518
+ tf_transform_resource_release(ledger, (uint64_t)canonical_len);
2519
+ return code;
2520
+ }
2521
+
2522
+ static const char *impute_op_name(tf_transform_impute_op op) {
2523
+ if (op == TF_TRANSFORM_IMPUTE_NONE) return "none";
2524
+ if (op == TF_TRANSFORM_IMPUTE_ZERO) return "zero";
2525
+ if (op == TF_TRANSFORM_IMPUTE_CONSTANT) return "constant";
2526
+ if (op == TF_TRANSFORM_IMPUTE_MEAN) return "mean";
2527
+ return "median";
2528
+ }
2529
+
2530
+ static const char *normalize_op_name(tf_transform_normalize_op op) {
2531
+ if (op == TF_TRANSFORM_NORMALIZE_NONE) return "none";
2532
+ if (op == TF_TRANSFORM_NORMALIZE_STANDARD) return "standard";
2533
+ return "minmax";
2534
+ }
2535
+
2536
+ static cJSON *numeric_step_to_json(
2537
+ const tf_transform_recipe_column *recipe,
2538
+ const tf_transform_numeric_state *state) {
2539
+ cJSON *step = cJSON_CreateObject();
2540
+ cJSON *numeric = cJSON_CreateObject();
2541
+ cJSON *impute = cJSON_CreateObject();
2542
+ cJSON *normalize = cJSON_CreateObject();
2543
+ if (!step || !numeric || !impute || !normalize) goto fail;
2544
+ if (!add_item(step, "categorical", json_null())
2545
+ || !add_item(step, "kind", json_string("numeric"))) goto fail;
2546
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_MEAN
2547
+ || recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
2548
+ if (!add_item(impute, "allMissing", json_string(
2549
+ recipe->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
2550
+ ? "zero" : "error"))) goto fail;
2551
+ } else if (!add_item(impute, "allMissing", json_null())) goto fail;
2552
+ if (!add_item(impute, "op", json_string(impute_op_name(recipe->impute))))
2553
+ goto fail;
2554
+ if (state->has_impute_value) {
2555
+ if (!add_item(impute, "value", tagged_constant(
2556
+ state->impute_value, TF_VIEW_FLOAT64))) goto fail;
2557
+ } else if (!add_item(impute, "value", json_null())) goto fail;
2558
+ if (!transfer_item(numeric, "impute", &impute)) goto fail;
2559
+ if (recipe->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
2560
+ if (!add_item(normalize, "ddof", json_number(recipe->ddof))) goto fail;
2561
+ } else if (!add_item(normalize, "ddof", json_null())) goto fail;
2562
+ if (!add_item(normalize, "location", tagged_constant(
2563
+ state->location, TF_VIEW_FLOAT64))
2564
+ || !add_item(normalize, "op", json_string(
2565
+ normalize_op_name(recipe->normalize)))
2566
+ || !add_item(normalize, "scale", tagged_constant(
2567
+ state->scale, TF_VIEW_FLOAT64))) goto fail;
2568
+ if (!transfer_item(numeric, "normalize", &normalize)) goto fail;
2569
+ if (!transfer_item(step, "numeric", &numeric)) goto fail;
2570
+ if (!add_item(step, "sourceId", json_string(recipe->source_id))
2571
+ || !add_item(step, "stateVersion", json_number(1))) goto fail;
2572
+ return step;
2573
+ fail:
2574
+ cJSON_Delete(step);
2575
+ cJSON_Delete(numeric);
2576
+ cJSON_Delete(impute);
2577
+ cJSON_Delete(normalize);
2578
+ return NULL;
2579
+ }
2580
+
2581
+ static cJSON *categorical_step_to_json(
2582
+ const tf_transform_recipe_column *recipe,
2583
+ const tf_transform_categorical_state *state) {
2584
+ cJSON *step = cJSON_CreateObject();
2585
+ cJSON *categorical = cJSON_CreateObject();
2586
+ cJSON *categories = cJSON_CreateArray();
2587
+ cJSON *encode = cJSON_CreateObject();
2588
+ cJSON *impute = cJSON_CreateObject();
2589
+ cJSON *other_ordinal = state->has_other_ordinal
2590
+ ? json_number((double)state->other_ordinal) : json_null();
2591
+ cJSON *sentinel = state->has_sentinel_label
2592
+ ? json_number((double)state->sentinel_label) : json_null();
2593
+ cJSON *unknown = state->unknown == TF_TRANSFORM_UNKNOWN_NONE
2594
+ ? json_null() : json_string(
2595
+ state->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? "sentinel"
2596
+ : state->unknown == TF_TRANSFORM_UNKNOWN_OTHER ? "other"
2597
+ : state->unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO
2598
+ ? "all_zero" : "error");
2599
+ cJSON *impute_all_missing
2600
+ = state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
2601
+ ? json_null() : json_string(
2602
+ recipe->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
2603
+ ? "zero" : "error");
2604
+ cJSON *impute_value
2605
+ = state->has_impute_value
2606
+ ? tagged_bits(state->impute_bits, state->source_dtype) : json_null();
2607
+ if (!step || !categorical || !categories || !encode || !impute
2608
+ || !other_ordinal || !sentinel || !unknown
2609
+ || !impute_all_missing || !impute_value) goto fail;
2610
+ for (size_t i = 0; i < state->category_count; ++i) {
2611
+ cJSON *tagged = tagged_bits(
2612
+ state->categories[i].bits, state->source_dtype);
2613
+ if (!tagged || !transfer_array_item(categories, &tagged)) {
2614
+ cJSON_Delete(tagged);
2615
+ goto fail;
2616
+ }
2617
+ }
2618
+ if (!transfer_item(categorical, "categories", &categories)
2619
+ || !add_item(encode, "op", json_string(
2620
+ state->encode == TF_TRANSFORM_ENCODE_LABEL ? "label"
2621
+ : state->encode == TF_TRANSFORM_ENCODE_ONEHOT ? "onehot" : "none"))
2622
+ || !transfer_item(encode, "otherOrdinal", &other_ordinal)
2623
+ || !transfer_item(encode, "sentinelLabel", &sentinel)
2624
+ || !transfer_item(encode, "unknown", &unknown)
2625
+ || !transfer_item(categorical, "encode", &encode)
2626
+ || !transfer_item(impute, "allMissing", &impute_all_missing)
2627
+ || !add_item(impute, "op", json_string(
2628
+ state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
2629
+ ? "none" : "mode"))
2630
+ || !transfer_item(impute, "value", &impute_value)
2631
+ || !transfer_item(categorical, "impute", &impute)
2632
+ || !transfer_item(step, "categorical", &categorical)
2633
+ || !add_item(step, "kind", json_string("categorical"))
2634
+ || !add_item(step, "numeric", json_null())
2635
+ || !add_item(step, "sourceId", json_string(recipe->source_id))
2636
+ || !add_item(step, "stateVersion", json_number(1))) goto fail;
2637
+ return step;
2638
+ fail:
2639
+ cJSON_Delete(step);
2640
+ cJSON_Delete(categorical);
2641
+ cJSON_Delete(categories);
2642
+ cJSON_Delete(encode);
2643
+ cJSON_Delete(impute);
2644
+ cJSON_Delete(other_ordinal);
2645
+ cJSON_Delete(sentinel);
2646
+ cJSON_Delete(unknown);
2647
+ cJSON_Delete(impute_all_missing);
2648
+ cJSON_Delete(impute_value);
2649
+ return NULL;
2650
+ }
2651
+
2652
+ cJSON *tf_transform_plan_to_json(
2653
+ const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
2654
+ tf_transform_error **error) {
2655
+ cJSON *root = NULL;
2656
+ cJSON *input_schema = NULL;
2657
+ cJSON *output_schema = NULL;
2658
+ cJSON *recipe = NULL;
2659
+ cJSON *steps = NULL;
2660
+ char fingerprint[65];
2661
+ tf_transform_code code;
2662
+ if (!plan || !limits) {
2663
+ tf_transform_set_error(
2664
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2665
+ "plan JSON argument is null");
2666
+ goto failed;
2667
+ }
2668
+ code = tf_transform_plan_json_preflight(plan, limits, error);
2669
+ if (code != TF_TRANSFORM_OK) goto failed;
2670
+ root = cJSON_CreateObject();
2671
+ steps = cJSON_CreateArray();
2672
+ if (!root || !steps) goto allocation_failed;
2673
+ code = tf_transform_plan_recipe_fingerprint(
2674
+ plan, limits, fingerprint, error);
2675
+ if (code != TF_TRANSFORM_OK) goto failed;
2676
+ input_schema = tf_transform_schema_to_json(&plan->input_schema);
2677
+ output_schema = tf_transform_schema_to_json(&plan->output_schema);
2678
+ recipe = tf_transform_recipe_to_json(plan->recipe);
2679
+ if (!input_schema || !output_schema || !recipe
2680
+ || !add_item(root, "format", json_string("tranfi.transform-plan"))
2681
+ || !transfer_item(root, "inputSchema", &input_schema)) goto allocation_failed;
2682
+ if (!transfer_item(root, "outputSchema", &output_schema)) goto allocation_failed;
2683
+ if (!transfer_item(root, "recipe", &recipe)) goto allocation_failed;
2684
+ if (!add_item(root, "recipeSha256", json_string(fingerprint)))
2685
+ goto allocation_failed;
2686
+ for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
2687
+ cJSON *step = plan->states[i].kind == TF_TRANSFORM_KIND_CATEGORICAL
2688
+ ? categorical_step_to_json(
2689
+ &plan->recipe->columns[i],
2690
+ &plan->states[i].value.categorical)
2691
+ : numeric_step_to_json(
2692
+ &plan->recipe->columns[i],
2693
+ &plan->states[i].value.numeric);
2694
+ if (!step || !transfer_array_item(steps, &step)) {
2695
+ cJSON_Delete(step);
2696
+ goto allocation_failed;
2697
+ }
2698
+ }
2699
+ if (!transfer_item(root, "steps", &steps)) goto allocation_failed;
2700
+ if (!add_item(root, "version", json_number(1))) goto allocation_failed;
2701
+ return root;
2702
+ allocation_failed:
2703
+ tf_transform_set_error(
2704
+ error, TF_TRANSFORM_ALLOCATION, "plan JSON allocation failed");
2705
+ failed:
2706
+ cJSON_Delete(input_schema);
2707
+ cJSON_Delete(output_schema);
2708
+ cJSON_Delete(recipe);
2709
+ cJSON_Delete(steps);
2710
+ cJSON_Delete(root);
2711
+ return NULL;
2712
+ }
2713
+
2714
+ static tf_transform_code parse_schema_json(
2715
+ const cJSON *array, int is_output,
2716
+ const tf_transform_runtime_copy *runtime,
2717
+ tf_transform_resource_ledger *ledger,
2718
+ tf_transform_schema *out, tf_transform_error **error) {
2719
+ static const char *const input_keys[] = {"dtype", "id", "name"};
2720
+ static const char *const output_keys[] = {
2721
+ "category", "dtype", "id", "name", "role", "sourceId"
2722
+ };
2723
+ tf_field_view_v1 *fields = NULL;
2724
+ tf_schema_view_v1 view;
2725
+ tf_transform_code code;
2726
+ size_t count = 0;
2727
+ const cJSON *entry;
2728
+ if (!cJSON_IsArray(array))
2729
+ return tf_transform_set_error(
2730
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema is not a nonempty array");
2731
+ if (!runtime) return tf_transform_set_error(
2732
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2733
+ "plan schema runtime is null");
2734
+ cJSON_ArrayForEach(entry, array) {
2735
+ if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
2736
+ code = tf_transform_poll_cancel(runtime, error);
2737
+ if (code != TF_TRANSFORM_OK) return code;
2738
+ }
2739
+ if (count == SIZE_MAX) return tf_transform_set_error(
2740
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
2741
+ "plan schema column count overflows");
2742
+ ++count;
2743
+ }
2744
+ if (count == 0) return tf_transform_set_error(
2745
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2746
+ "plan schema is not a nonempty array");
2747
+ if ((uint64_t)count > (is_output ? runtime->limits.max_output_columns
2748
+ : runtime->limits.max_input_columns)
2749
+ || (size_t)count > SIZE_MAX / sizeof(*fields)
2750
+ || (uint64_t)count * sizeof(*fields)
2751
+ > runtime->limits.max_allocation_bytes)
2752
+ return tf_transform_set_error(
2753
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "plan schema exceeds limits");
2754
+ code = tf_transform_poll_cancel(runtime, error);
2755
+ if (code != TF_TRANSFORM_OK) return code;
2756
+ fields = (tf_field_view_v1 *)tf_transform_resource_calloc(
2757
+ ledger, count, sizeof(*fields), error);
2758
+ if (!fields) return ledger->last_code != TF_TRANSFORM_OK
2759
+ ? ledger->last_code : tf_transform_set_error(
2760
+ error, TF_TRANSFORM_ALLOCATION,
2761
+ "plan schema view allocation failed");
2762
+ entry = array->child;
2763
+ for (size_t i = 0; i < count; ++i, entry = entry->next) {
2764
+ const cJSON *dtype;
2765
+ const cJSON *id;
2766
+ const cJSON *name;
2767
+ size_t id_len;
2768
+ size_t name_len;
2769
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
2770
+ code = tf_transform_poll_cancel(runtime, error);
2771
+ if (code != TF_TRANSFORM_OK) goto done;
2772
+ }
2773
+ if (!exact_keys(entry, is_output ? output_keys : input_keys,
2774
+ is_output ? 6 : 3)) {
2775
+ code = tf_transform_set_error(
2776
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema field shape is invalid");
2777
+ goto done;
2778
+ }
2779
+ dtype = required_item(entry, "dtype");
2780
+ id = required_item(entry, "id");
2781
+ name = required_item(entry, "name");
2782
+ if ((!json_string_equals(dtype, "float32")
2783
+ && !json_string_equals(dtype, "float64"))
2784
+ || !cJSON_IsString(id) || !id->valuestring
2785
+ || !cJSON_IsString(name) || !name->valuestring) {
2786
+ code = tf_transform_set_error(
2787
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema field is invalid");
2788
+ goto done;
2789
+ }
2790
+ if (is_output) {
2791
+ static const char *const other_keys[] = {"t"};
2792
+ const cJSON *category = required_item(entry, "category");
2793
+ const cJSON *role = required_item(entry, "role");
2794
+ const cJSON *source_id = required_item(entry, "sourceId");
2795
+ uint64_t category_bits = 0;
2796
+ uint32_t category_dtype = 0;
2797
+ size_t source_id_len = 0;
2798
+ if (cJSON_IsString(source_id) && source_id->valuestring) {
2799
+ code = runtime_cstring_length(
2800
+ source_id->valuestring,
2801
+ (size_t)runtime->limits.max_string_bytes,
2802
+ runtime, &source_id_len, error);
2803
+ if (code != TF_TRANSFORM_OK) goto done;
2804
+ }
2805
+ code = runtime_cstring_length(
2806
+ id->valuestring, (size_t)runtime->limits.max_string_bytes,
2807
+ runtime, &id_len, error);
2808
+ if (code != TF_TRANSFORM_OK) goto done;
2809
+ if ((!json_string_equals(role, "value")
2810
+ && !json_string_equals(role, "label")
2811
+ && !json_string_equals(role, "onehot"))
2812
+ || !cJSON_IsString(source_id) || !source_id->valuestring
2813
+ || !json_string_equals(dtype, "float64")) {
2814
+ code = tf_transform_set_error(
2815
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2816
+ "output schema metadata is invalid");
2817
+ goto done;
2818
+ }
2819
+ if (json_string_equals(role, "onehot")) {
2820
+ if (!parse_tagged_bits(
2821
+ category, &category_bits, &category_dtype)
2822
+ && !(exact_keys(category, other_keys, 1)
2823
+ && json_string_equals(
2824
+ required_item(category, "t"), "other"))) {
2825
+ code = tf_transform_set_error(
2826
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2827
+ "one-hot output category is invalid");
2828
+ goto done;
2829
+ }
2830
+ } else if (!cJSON_IsNull(category)) {
2831
+ code = tf_transform_set_error(
2832
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2833
+ "non-one-hot output category is invalid");
2834
+ goto done;
2835
+ }
2836
+ }
2837
+ fields[i].abi_version = 1;
2838
+ fields[i].struct_size = (uint32_t)sizeof(fields[i]);
2839
+ fields[i].dtype = json_string_equals(dtype, "float32")
2840
+ ? TF_VIEW_FLOAT32 : TF_VIEW_FLOAT64;
2841
+ if (!is_output) {
2842
+ code = runtime_cstring_length(
2843
+ id->valuestring, (size_t)runtime->limits.max_string_bytes,
2844
+ runtime, &id_len, error);
2845
+ if (code != TF_TRANSFORM_OK) goto done;
2846
+ }
2847
+ code = runtime_cstring_length(
2848
+ name->valuestring, (size_t)runtime->limits.max_string_bytes,
2849
+ runtime, &name_len, error);
2850
+ if (code != TF_TRANSFORM_OK) goto done;
2851
+ fields[i].id_utf8 = (const uint8_t *)id->valuestring;
2852
+ fields[i].id_bytes = id_len;
2853
+ fields[i].name_utf8 = (const uint8_t *)name->valuestring;
2854
+ fields[i].name_bytes = name_len;
2855
+ }
2856
+ memset(&view, 0, sizeof(view));
2857
+ view.abi_version = 1;
2858
+ view.struct_size = (uint32_t)sizeof(view);
2859
+ view.column_count = count;
2860
+ view.fields = fields;
2861
+ view.fields_bytes = count * sizeof(*fields);
2862
+ code = tf_transform_schema_copy_runtime_ledger(
2863
+ &view, runtime, ledger, out, error);
2864
+ if (code == TF_TRANSFORM_INVALID_ARGUMENT)
2865
+ code = tf_transform_set_error(
2866
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema validation failed");
2867
+ if (code == TF_TRANSFORM_OK) out->is_output = is_output;
2868
+ if (code == TF_TRANSFORM_OK && is_output) {
2869
+ entry = array->child;
2870
+ for (size_t i = 0; i < count; ++i, entry = entry->next) {
2871
+ static const char *const other_keys[] = {"t"};
2872
+ const cJSON *category = required_item(entry, "category");
2873
+ const cJSON *role = required_item(entry, "role");
2874
+ const cJSON *source_id = required_item(entry, "sourceId");
2875
+ size_t source_id_len = 0;
2876
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
2877
+ code = tf_transform_poll_cancel(runtime, error);
2878
+ if (code != TF_TRANSFORM_OK) break;
2879
+ }
2880
+ code = runtime_cstring_length(
2881
+ source_id->valuestring,
2882
+ (size_t)runtime->limits.max_string_bytes,
2883
+ runtime, &source_id_len, error);
2884
+ if (code != TF_TRANSFORM_OK) break;
2885
+ out->fields[i].source_id = (char *)tf_transform_resource_malloc(
2886
+ ledger, source_id_len + 1, error);
2887
+ if (!out->fields[i].source_id) {
2888
+ code = ledger->last_code != TF_TRANSFORM_OK
2889
+ ? ledger->last_code : tf_transform_set_error(
2890
+ error, TF_TRANSFORM_ALLOCATION,
2891
+ "output schema source ID allocation failed");
2892
+ break;
2893
+ }
2894
+ code = tf_transform_copy_bytes_runtime(
2895
+ out->fields[i].source_id, source_id->valuestring,
2896
+ source_id_len + 1, runtime, error);
2897
+ if (code != TF_TRANSFORM_OK) break;
2898
+ out->fields[i].source_id_len = source_id_len;
2899
+ if (json_string_equals(role, "label"))
2900
+ out->fields[i].role = TF_TRANSFORM_ROLE_LABEL;
2901
+ else if (json_string_equals(role, "onehot"))
2902
+ out->fields[i].role = TF_TRANSFORM_ROLE_ONEHOT;
2903
+ else out->fields[i].role = TF_TRANSFORM_ROLE_VALUE;
2904
+ if (out->fields[i].role == TF_TRANSFORM_ROLE_ONEHOT) {
2905
+ uint64_t bits = 0;
2906
+ uint32_t category_dtype = 0;
2907
+ if (parse_tagged_bits(category, &bits, &category_dtype)) {
2908
+ out->fields[i].category_kind
2909
+ = TF_TRANSFORM_SCHEMA_CATEGORY_VALUE;
2910
+ out->fields[i].category_dtype = category_dtype;
2911
+ out->fields[i].category_bits = bits;
2912
+ } else if (exact_keys(category, other_keys, 1)
2913
+ && json_string_equals(
2914
+ required_item(category, "t"), "other")) {
2915
+ out->fields[i].category_kind
2916
+ = TF_TRANSFORM_SCHEMA_CATEGORY_OTHER;
2917
+ } else {
2918
+ code = tf_transform_set_error(
2919
+ error, TF_TRANSFORM_CORRUPT_PLAN,
2920
+ "one-hot output category is invalid");
2921
+ break;
2922
+ }
2923
+ } else {
2924
+ out->fields[i].category_kind
2925
+ = TF_TRANSFORM_SCHEMA_CATEGORY_NONE;
2926
+ }
2927
+ }
2928
+ if (code != TF_TRANSFORM_OK) tf_transform_schema_clear(out);
2929
+ }
2930
+ done:
2931
+ free(fields);
2932
+ tf_transform_resource_release(
2933
+ ledger, (uint64_t)count * sizeof(*fields));
2934
+ return code;
2935
+ }
2936
+
2937
+ static tf_transform_code lowercase_sha256_string_runtime(
2938
+ const cJSON *value, const tf_transform_runtime_copy *runtime,
2939
+ int *valid, tf_transform_error **error) {
2940
+ size_t length = 0;
2941
+ tf_transform_code code;
2942
+ if (!valid) return tf_transform_set_error(
2943
+ error, TF_TRANSFORM_INVALID_ARGUMENT,
2944
+ "fingerprint validation output is null");
2945
+ *valid = 0;
2946
+ if (!cJSON_IsString(value) || !value->valuestring) return TF_TRANSFORM_OK;
2947
+ code = runtime_cstring_length(
2948
+ value->valuestring, (size_t)runtime->limits.max_string_bytes,
2949
+ runtime, &length, error);
2950
+ if (code != TF_TRANSFORM_OK) return code;
2951
+ if (length != 64) return TF_TRANSFORM_OK;
2952
+ for (size_t i = 0; i < 64; ++i) {
2953
+ unsigned char c = (unsigned char)value->valuestring[i];
2954
+ if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')))
2955
+ return TF_TRANSFORM_OK;
2956
+ }
2957
+ *valid = 1;
2958
+ return TF_TRANSFORM_OK;
2959
+ }
2960
+
2961
+ static int tagged_f64(const cJSON *value, double *out) {
2962
+ uint32_t dtype = 0;
2963
+ return parse_tagged_constant(value, out, &dtype) && dtype == TF_VIEW_FLOAT64;
2964
+ }
2965
+
2966
+ static int tagged_category_bits(
2967
+ const cJSON *value, uint32_t expected_dtype, uint64_t *out) {
2968
+ uint64_t decoded = 0;
2969
+ uint32_t dtype = 0;
2970
+ if (!out || !parse_tagged_bits(value, &decoded, &dtype)
2971
+ || dtype != expected_dtype)
2972
+ return 0;
2973
+ if (dtype == TF_VIEW_FLOAT32) {
2974
+ uint32_t bits = (uint32_t)decoded;
2975
+ if ((bits & UINT32_C(0x7f800000)) == UINT32_C(0x7f800000)
2976
+ || bits == UINT32_C(0x80000000)) return 0;
2977
+ *out = bits;
2978
+ return 1;
2979
+ }
2980
+ if ((decoded & UINT64_C(0x7ff0000000000000))
2981
+ == UINT64_C(0x7ff0000000000000)
2982
+ || decoded == UINT64_C(0x8000000000000000)) return 0;
2983
+ *out = decoded;
2984
+ return 1;
2985
+ }
2986
+
2987
+ static int category_bits_are_integer(uint64_t bits, uint32_t dtype) {
2988
+ double integer;
2989
+ double value = tf_transform_category_decode(bits, dtype);
2990
+ return modf(value, &integer) == 0.0;
2991
+ }
2992
+
2993
+ static int null_or_policy(
2994
+ const cJSON *value, tf_transform_all_missing policy,
2995
+ tf_transform_impute_op op) {
2996
+ if (op != TF_TRANSFORM_IMPUTE_MEAN
2997
+ && op != TF_TRANSFORM_IMPUTE_MEDIAN) return cJSON_IsNull(value);
2998
+ return json_string_equals(value,
2999
+ policy == TF_TRANSFORM_ALL_MISSING_ZERO ? "zero" : "error");
3000
+ }
3001
+
3002
+ static tf_transform_code parse_numeric_step(
3003
+ const cJSON *value, const tf_transform_recipe_column *recipe,
3004
+ const tf_transform_runtime_copy *runtime,
3005
+ tf_transform_numeric_state *out, tf_transform_error **error) {
3006
+ static const char *const step_keys[] = {
3007
+ "categorical", "kind", "numeric", "sourceId", "stateVersion"
3008
+ };
3009
+ static const char *const numeric_keys[] = {"impute", "normalize"};
3010
+ static const char *const impute_keys[] = {"allMissing", "op", "value"};
3011
+ static const char *const normalize_keys[] = {
3012
+ "ddof", "location", "op", "scale"
3013
+ };
3014
+ const cJSON *numeric;
3015
+ const cJSON *impute;
3016
+ const cJSON *normalize;
3017
+ const cJSON *impute_value;
3018
+ const cJSON *ddof;
3019
+ uint64_t state_version;
3020
+ uint64_t parsed_ddof;
3021
+ double location;
3022
+ double scale;
3023
+ double resolved = 0.0;
3024
+ const cJSON *source_id = required_item(value, "sourceId");
3025
+ size_t source_id_len = 0;
3026
+ int source_equal = 0;
3027
+ tf_transform_code code;
3028
+ memset(out, 0, sizeof(*out));
3029
+ if (cJSON_IsString(source_id) && source_id->valuestring) {
3030
+ code = runtime_cstring_length(
3031
+ source_id->valuestring,
3032
+ (size_t)runtime->limits.max_string_bytes,
3033
+ runtime, &source_id_len, error);
3034
+ if (code != TF_TRANSFORM_OK) return code;
3035
+ code = runtime_span_equal(
3036
+ source_id->valuestring, source_id_len,
3037
+ recipe->source_id, recipe->source_id_len,
3038
+ runtime, &source_equal, error);
3039
+ if (code != TF_TRANSFORM_OK) return code;
3040
+ }
3041
+ if (!exact_keys(value, step_keys, 5)
3042
+ || !cJSON_IsNull(required_item(value, "categorical"))
3043
+ || !json_string_equals(required_item(value, "kind"), "numeric")
3044
+ || !source_equal
3045
+ || !safe_uint64(required_item(value, "stateVersion"), &state_version))
3046
+ return tf_transform_set_error(
3047
+ error, TF_TRANSFORM_CORRUPT_PLAN, "numeric plan step shape is invalid");
3048
+ if (state_version != 1)
3049
+ return tf_transform_set_error(
3050
+ error, TF_TRANSFORM_UNSUPPORTED_VERSION,
3051
+ "numeric plan state version is unsupported");
3052
+ numeric = required_item(value, "numeric");
3053
+ if (!exact_keys(numeric, numeric_keys, 2)) goto corrupt;
3054
+ impute = required_item(numeric, "impute");
3055
+ normalize = required_item(numeric, "normalize");
3056
+ if (!exact_keys(impute, impute_keys, 3)
3057
+ || !json_string_equals(required_item(impute, "op"),
3058
+ impute_op_name(recipe->impute))
3059
+ || !null_or_policy(required_item(impute, "allMissing"),
3060
+ recipe->all_missing, recipe->impute)) goto corrupt;
3061
+ impute_value = required_item(impute, "value");
3062
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_NONE) {
3063
+ if (!cJSON_IsNull(impute_value)) goto corrupt;
3064
+ } else {
3065
+ if (!tagged_f64(impute_value, &resolved)) goto corrupt;
3066
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_ZERO
3067
+ && tf_transform_double_bits(resolved) != 0) goto corrupt;
3068
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN
3069
+ && resolved == 0.0
3070
+ && tf_transform_double_bits(resolved) != 0) goto corrupt;
3071
+ if (recipe->impute == TF_TRANSFORM_IMPUTE_CONSTANT
3072
+ && tf_transform_double_bits(resolved)
3073
+ != tf_transform_double_bits(recipe->constant)) goto corrupt;
3074
+ out->has_impute_value = 1;
3075
+ out->impute_value = resolved;
3076
+ }
3077
+ if (!exact_keys(normalize, normalize_keys, 4)
3078
+ || !json_string_equals(required_item(normalize, "op"),
3079
+ normalize_op_name(recipe->normalize))
3080
+ || !tagged_f64(required_item(normalize, "location"), &location)
3081
+ || !tagged_f64(required_item(normalize, "scale"), &scale)
3082
+ || scale <= 0.0) goto corrupt;
3083
+ ddof = required_item(normalize, "ddof");
3084
+ if (recipe->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
3085
+ if (!safe_uint64(ddof, &parsed_ddof)
3086
+ || parsed_ddof != (uint64_t)recipe->ddof) goto corrupt;
3087
+ } else if (!cJSON_IsNull(ddof)) goto corrupt;
3088
+ if (recipe->normalize == TF_TRANSFORM_NORMALIZE_NONE
3089
+ && (tf_transform_double_bits(location) != 0
3090
+ || tf_transform_double_bits(scale) != UINT64_C(0x3ff0000000000000)))
3091
+ goto corrupt;
3092
+ out->impute = recipe->impute;
3093
+ out->all_missing = recipe->all_missing;
3094
+ out->normalize = recipe->normalize;
3095
+ out->ddof = recipe->ddof;
3096
+ out->location = location;
3097
+ out->scale = scale;
3098
+ return TF_TRANSFORM_OK;
3099
+ corrupt:
3100
+ return tf_transform_set_error(
3101
+ error, TF_TRANSFORM_CORRUPT_PLAN, "numeric plan state is inconsistent");
3102
+ }
3103
+
3104
+ static tf_transform_code parse_categorical_step(
3105
+ const cJSON *value, const tf_transform_recipe_column *recipe,
3106
+ uint32_t source_dtype, const tf_transform_runtime_copy *runtime,
3107
+ tf_transform_resource_ledger *ledger, uint64_t *total_categories,
3108
+ tf_transform_categorical_state *out, tf_transform_error **error) {
3109
+ static const char *const step_keys[] = {
3110
+ "categorical", "kind", "numeric", "sourceId", "stateVersion"
3111
+ };
3112
+ static const char *const categorical_keys[] = {
3113
+ "categories", "encode", "impute"
3114
+ };
3115
+ static const char *const encode_keys[] = {
3116
+ "op", "otherOrdinal", "sentinelLabel", "unknown"
3117
+ };
3118
+ static const char *const impute_keys[] = {"allMissing", "op", "value"};
3119
+ const cJSON *categorical;
3120
+ const cJSON *categories;
3121
+ const cJSON *encode;
3122
+ const cJSON *impute;
3123
+ const cJSON *entry;
3124
+ const cJSON *source_id = required_item(value, "sourceId");
3125
+ size_t source_id_len = 0;
3126
+ size_t count = 0;
3127
+ size_t bytes = 0;
3128
+ uint64_t state_version;
3129
+ uint64_t mode_bits;
3130
+ uint64_t other_ordinal = 0;
3131
+ uint64_t previous_category_bits = 0;
3132
+ int64_t sentinel_label = 0;
3133
+ int empty_categories_allowed = 0;
3134
+ int source_equal = 0;
3135
+ int mode_known = 0;
3136
+ tf_transform_code code;
3137
+ memset(out, 0, sizeof(*out));
3138
+ if (cJSON_IsString(source_id) && source_id->valuestring) {
3139
+ code = runtime_cstring_length(
3140
+ source_id->valuestring, (size_t)runtime->limits.max_string_bytes,
3141
+ runtime, &source_id_len, error);
3142
+ if (code != TF_TRANSFORM_OK) return code;
3143
+ code = runtime_span_equal(
3144
+ source_id->valuestring, source_id_len,
3145
+ recipe->source_id, recipe->source_id_len,
3146
+ runtime, &source_equal, error);
3147
+ if (code != TF_TRANSFORM_OK) return code;
3148
+ }
3149
+ if (!exact_keys(value, step_keys, 5)
3150
+ || !json_string_equals(required_item(value, "kind"), "categorical")
3151
+ || !cJSON_IsNull(required_item(value, "numeric"))
3152
+ || !source_equal
3153
+ || !safe_uint64(required_item(value, "stateVersion"), &state_version))
3154
+ goto corrupt;
3155
+ if (state_version != 1)
3156
+ return tf_transform_set_error(
3157
+ error, TF_TRANSFORM_UNSUPPORTED_VERSION,
3158
+ "categorical plan state version is unsupported");
3159
+ categorical = required_item(value, "categorical");
3160
+ if (!exact_keys(categorical, categorical_keys, 3)) goto corrupt;
3161
+ categories = required_item(categorical, "categories");
3162
+ encode = required_item(categorical, "encode");
3163
+ impute = required_item(categorical, "impute");
3164
+ if (!cJSON_IsArray(categories)
3165
+ || !exact_keys(encode, encode_keys, 4)
3166
+ || !exact_keys(impute, impute_keys, 3))
3167
+ goto corrupt;
3168
+ if (recipe->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
3169
+ if (!json_string_equals(required_item(impute, "op"), "none")
3170
+ || !cJSON_IsNull(required_item(impute, "allMissing"))
3171
+ || !cJSON_IsNull(required_item(impute, "value"))) goto corrupt;
3172
+ } else if (recipe->categorical_impute
3173
+ == TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE) {
3174
+ if (!json_string_equals(required_item(impute, "op"), "mode")
3175
+ || !json_string_equals(
3176
+ required_item(impute, "allMissing"),
3177
+ recipe->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
3178
+ ? "zero" : "error")) goto corrupt;
3179
+ } else goto corrupt;
3180
+ if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_NONE) {
3181
+ if (!json_string_equals(required_item(encode, "op"), "none")
3182
+ || !cJSON_IsNull(required_item(encode, "otherOrdinal"))
3183
+ || !cJSON_IsNull(required_item(encode, "sentinelLabel"))
3184
+ || !cJSON_IsNull(required_item(encode, "unknown"))) goto corrupt;
3185
+ } else if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_LABEL) {
3186
+ if (!json_string_equals(required_item(encode, "op"), "label")) goto corrupt;
3187
+ if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_ERROR) {
3188
+ if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
3189
+ || !cJSON_IsNull(required_item(encode, "sentinelLabel"))
3190
+ || !json_string_equals(required_item(encode, "unknown"), "error"))
3191
+ goto corrupt;
3192
+ } else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL) {
3193
+ if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
3194
+ || !safe_int64(
3195
+ required_item(encode, "sentinelLabel"), &sentinel_label)
3196
+ || sentinel_label != recipe->categorical_sentinel_label
3197
+ || !json_string_equals(
3198
+ required_item(encode, "unknown"), "sentinel")) goto corrupt;
3199
+ } else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
3200
+ if (!safe_uint64(
3201
+ required_item(encode, "otherOrdinal"), &other_ordinal)
3202
+ || !cJSON_IsNull(required_item(encode, "sentinelLabel"))
3203
+ || !json_string_equals(required_item(encode, "unknown"), "other"))
3204
+ goto corrupt;
3205
+ } else goto corrupt;
3206
+ } else if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT) {
3207
+ if (!json_string_equals(required_item(encode, "op"), "onehot")
3208
+ || !cJSON_IsNull(required_item(encode, "sentinelLabel")))
3209
+ goto corrupt;
3210
+ if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_ERROR) {
3211
+ if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
3212
+ || !json_string_equals(
3213
+ required_item(encode, "unknown"), "error")) goto corrupt;
3214
+ } else if (recipe->categorical_unknown
3215
+ == TF_TRANSFORM_UNKNOWN_ALL_ZERO) {
3216
+ if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
3217
+ || !json_string_equals(
3218
+ required_item(encode, "unknown"), "all_zero"))
3219
+ goto corrupt;
3220
+ } else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
3221
+ if (!safe_uint64(
3222
+ required_item(encode, "otherOrdinal"), &other_ordinal)
3223
+ || !json_string_equals(
3224
+ required_item(encode, "unknown"), "other")) goto corrupt;
3225
+ } else goto corrupt;
3226
+ } else goto corrupt;
3227
+ cJSON_ArrayForEach(entry, categories) {
3228
+ uint64_t bits;
3229
+ if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3230
+ code = tf_transform_poll_cancel(runtime, error);
3231
+ if (code != TF_TRANSFORM_OK) return code;
3232
+ }
3233
+ if (!tagged_category_bits(entry, source_dtype, &bits)
3234
+ || (recipe->kind == TF_TRANSFORM_KIND_INFER
3235
+ && !category_bits_are_integer(bits, source_dtype))
3236
+ || (count != 0 && tf_transform_category_compare(
3237
+ previous_category_bits, bits, source_dtype) >= 0))
3238
+ goto corrupt;
3239
+ if (recipe->categorical_fixed_count
3240
+ && (count >= recipe->categorical_fixed_count
3241
+ || bits != recipe->categorical_fixed[count].bits)) goto corrupt;
3242
+ previous_category_bits = bits;
3243
+ if (count == SIZE_MAX) return tf_transform_set_error(
3244
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3245
+ "categorical plan category count overflows");
3246
+ ++count;
3247
+ }
3248
+ empty_categories_allowed
3249
+ = recipe->kind == TF_TRANSFORM_KIND_CATEGORICAL
3250
+ && recipe->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
3251
+ && recipe->categorical_encode == TF_TRANSFORM_ENCODE_NONE;
3252
+ if (count == 0 && !empty_categories_allowed) goto corrupt;
3253
+ if (count != 0 && empty_categories_allowed) goto corrupt;
3254
+ if (recipe->kind == TF_TRANSFORM_KIND_INFER
3255
+ && (count < 2 || (uint64_t)count > recipe->infer_max_categories))
3256
+ goto corrupt;
3257
+ if (recipe->categorical_fixed_count && count != recipe->categorical_fixed_count)
3258
+ goto corrupt;
3259
+ if ((uint64_t)count > runtime->limits.max_categories_per_column
3260
+ || *total_categories > UINT64_MAX - (uint64_t)count
3261
+ || *total_categories + (uint64_t)count
3262
+ > runtime->limits.max_total_categories
3263
+ || count > SIZE_MAX / sizeof(*out->categories))
3264
+ return tf_transform_set_error(
3265
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3266
+ "categorical plan state exceeds category limits");
3267
+ if ((recipe->categorical_encode == TF_TRANSFORM_ENCODE_LABEL
3268
+ || recipe->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT)
3269
+ && (uint64_t)count > (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1)
3270
+ return tf_transform_set_error(
3271
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3272
+ "categorical encoding count exceeds the safe-integer domain");
3273
+ bytes = count * sizeof(*out->categories);
3274
+ if ((uint64_t)bytes > runtime->limits.max_allocation_bytes)
3275
+ return tf_transform_set_error(
3276
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3277
+ "categorical plan allocation exceeds limits");
3278
+ if (count != 0) {
3279
+ out->categories = (tf_transform_category_value *)
3280
+ tf_transform_resource_calloc(
3281
+ ledger, count, sizeof(*out->categories), error);
3282
+ if (!out->categories)
3283
+ return ledger->last_code != TF_TRANSFORM_OK
3284
+ ? ledger->last_code : tf_transform_set_error(
3285
+ error, TF_TRANSFORM_ALLOCATION,
3286
+ "categorical plan state allocation failed");
3287
+ }
3288
+ out->source_dtype = source_dtype;
3289
+ out->category_count = count;
3290
+ entry = categories->child;
3291
+ for (size_t i = 0; i < count; ++i, entry = entry->next) {
3292
+ uint64_t bits;
3293
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3294
+ code = tf_transform_poll_cancel(runtime, error);
3295
+ if (code != TF_TRANSFORM_OK) goto failed;
3296
+ }
3297
+ if (!tagged_category_bits(entry, source_dtype, &bits)
3298
+ || (recipe->kind == TF_TRANSFORM_KIND_INFER
3299
+ && !category_bits_are_integer(bits, source_dtype))
3300
+ || (i != 0 && tf_transform_category_compare(
3301
+ out->categories[i - 1].bits, bits, source_dtype) >= 0))
3302
+ goto corrupt_allocated;
3303
+ out->categories[i].bits = bits;
3304
+ }
3305
+ out->impute = recipe->categorical_impute;
3306
+ out->all_missing = recipe->categorical_all_missing;
3307
+ out->encode = recipe->categorical_encode;
3308
+ out->unknown = recipe->categorical_unknown;
3309
+ if (out->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL) {
3310
+ if (sentinel_label >= 0 && (uint64_t)sentinel_label < (uint64_t)count)
3311
+ goto corrupt_allocated;
3312
+ out->sentinel_label = sentinel_label;
3313
+ out->has_sentinel_label = 1;
3314
+ } else if (out->unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
3315
+ if (other_ordinal != (uint64_t)count) goto corrupt_allocated;
3316
+ out->other_ordinal = other_ordinal;
3317
+ out->has_other_ordinal = 1;
3318
+ }
3319
+ if (out->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE) {
3320
+ if (!tagged_category_bits(
3321
+ required_item(impute, "value"), source_dtype, &mode_bits))
3322
+ goto corrupt_allocated;
3323
+ out->impute_bits = mode_bits;
3324
+ out->has_impute_value = 1;
3325
+ {
3326
+ size_t mode_ordinal = 0;
3327
+ code = tf_transform_category_lookup(
3328
+ out, mode_bits, runtime, &mode_ordinal, &mode_known, error);
3329
+ }
3330
+ if (code != TF_TRANSFORM_OK) goto failed;
3331
+ if (!mode_known) goto corrupt_allocated;
3332
+ }
3333
+ *total_categories += (uint64_t)count;
3334
+ return TF_TRANSFORM_OK;
3335
+ corrupt_allocated:
3336
+ code = tf_transform_set_error(
3337
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3338
+ "categorical plan state is inconsistent");
3339
+ failed:
3340
+ tf_transform_categorical_state_clear(out);
3341
+ tf_transform_resource_release(ledger, (uint64_t)bytes);
3342
+ return code;
3343
+ corrupt:
3344
+ return tf_transform_set_error(
3345
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3346
+ "categorical plan state is inconsistent");
3347
+ }
3348
+
3349
+ static tf_transform_code imported_step_kind(
3350
+ const cJSON *step, const tf_transform_recipe_column *recipe,
3351
+ tf_transform_column_kind *out, tf_transform_error **error) {
3352
+ const cJSON *kind;
3353
+ tf_transform_column_kind resolved;
3354
+ if (!step || !recipe || !out)
3355
+ return tf_transform_set_error(
3356
+ error, TF_TRANSFORM_INTERNAL,
3357
+ "imported step-kind arguments are invalid");
3358
+ kind = required_item(step, "kind");
3359
+ if (json_string_equals(kind, "numeric"))
3360
+ resolved = TF_TRANSFORM_KIND_NUMERIC;
3361
+ else if (json_string_equals(kind, "categorical"))
3362
+ resolved = TF_TRANSFORM_KIND_CATEGORICAL;
3363
+ else return tf_transform_set_error(
3364
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3365
+ "plan step kind is invalid");
3366
+ if ((recipe->kind == TF_TRANSFORM_KIND_NUMERIC
3367
+ && resolved != TF_TRANSFORM_KIND_NUMERIC)
3368
+ || (recipe->kind == TF_TRANSFORM_KIND_CATEGORICAL
3369
+ && resolved != TF_TRANSFORM_KIND_CATEGORICAL)
3370
+ || (recipe->kind != TF_TRANSFORM_KIND_NUMERIC
3371
+ && recipe->kind != TF_TRANSFORM_KIND_CATEGORICAL
3372
+ && recipe->kind != TF_TRANSFORM_KIND_INFER))
3373
+ return tf_transform_set_error(
3374
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3375
+ "plan step kind disagrees with its recipe");
3376
+ *out = resolved;
3377
+ return TF_TRANSFORM_OK;
3378
+ }
3379
+
3380
+ tf_transform_code tf_transform_plan_from_json(
3381
+ const uint8_t *json, size_t json_len,
3382
+ const tf_transform_runtime_copy *runtime,
3383
+ tf_transform_plan **out, uint8_t **canonical_out,
3384
+ size_t *canonical_len_out, tf_transform_error **error) {
3385
+ static const char *const top_keys[] = {
3386
+ "format", "inputSchema", "outputSchema", "recipe",
3387
+ "recipeSha256", "steps", "version"
3388
+ };
3389
+ cJSON *root = NULL;
3390
+ tf_transform_recipe *recipe = NULL;
3391
+ tf_transform_plan *plan = NULL;
3392
+ tf_transform_resource_ledger ledger;
3393
+ tf_transform_runtime_copy parse_runtime;
3394
+ tf_transform_runtime_copy recipe_runtime;
3395
+ tf_transform_code code;
3396
+ uint64_t version;
3397
+ char expected_fingerprint[65];
3398
+ const cJSON *steps;
3399
+ size_t step_count = 0;
3400
+ size_t output_column_count = 0;
3401
+ uint64_t total_categories = 0;
3402
+ uint64_t ast_resident_bytes = 0;
3403
+ if (out) *out = NULL;
3404
+ if (canonical_out) *canonical_out = NULL;
3405
+ if (canonical_len_out) *canonical_len_out = 0;
3406
+ if (!json || json_len == 0 || !runtime || !out
3407
+ || !canonical_out || !canonical_len_out)
3408
+ return tf_transform_set_error(
3409
+ error, TF_TRANSFORM_INVALID_ARGUMENT, "plan JSON argument is null");
3410
+ parse_runtime = *runtime;
3411
+ parse_runtime.limits.max_categories_per_column
3412
+ = (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
3413
+ parse_runtime.limits.max_total_categories
3414
+ = (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
3415
+ tf_transform_resource_ledger_init(&ledger, &parse_runtime);
3416
+ code = tf_transform_json_parse_bounded_runtime_ledger(
3417
+ json, json_len, &parse_runtime, &ledger, &root,
3418
+ &ast_resident_bytes, NULL, error,
3419
+ TF_TRANSFORM_CORRUPT_PLAN);
3420
+ if (code != TF_TRANSFORM_OK) return code;
3421
+ ledger.runtime = runtime;
3422
+ code = tf_transform_poll_cancel(runtime, error);
3423
+ if (code != TF_TRANSFORM_OK) goto done;
3424
+ if (!exact_keys(root, top_keys, 7)
3425
+ || !json_string_equals(required_item(root, "format"),
3426
+ "tranfi.transform-plan")
3427
+ || !safe_uint64(required_item(root, "version"), &version)) {
3428
+ code = tf_transform_set_error(
3429
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan JSON header is invalid");
3430
+ goto done;
3431
+ }
3432
+ if (version != 1) {
3433
+ code = tf_transform_set_error(
3434
+ error, TF_TRANSFORM_UNSUPPORTED_VERSION, "plan JSON version is unsupported");
3435
+ goto done;
3436
+ }
3437
+ {
3438
+ const cJSON *embedded_recipe = required_item(root, "recipe");
3439
+ uint64_t recipe_version;
3440
+ uint64_t policy_version;
3441
+ if (cJSON_IsObject(embedded_recipe)
3442
+ && safe_uint64(required_item(embedded_recipe, "version"),
3443
+ &recipe_version)
3444
+ && safe_uint64(required_item(embedded_recipe, "policyVersion"),
3445
+ &policy_version)
3446
+ && (recipe_version != 1 || policy_version != 1)) {
3447
+ code = tf_transform_set_error(
3448
+ error, TF_TRANSFORM_UNSUPPORTED_VERSION,
3449
+ "embedded recipe version is unsupported");
3450
+ goto done;
3451
+ }
3452
+ }
3453
+ recipe_runtime = *runtime;
3454
+ recipe_runtime.limits.max_output_columns
3455
+ = (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
3456
+ recipe_runtime.limits.max_categories_per_column
3457
+ = (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
3458
+ recipe_runtime.limits.max_total_categories
3459
+ = (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
3460
+ code = recipe_from_json_value(
3461
+ required_item(root, "recipe"), &recipe_runtime,
3462
+ &ledger, &recipe, error);
3463
+ if (code != TF_TRANSFORM_OK) {
3464
+ if (code == TF_TRANSFORM_INVALID_RECIPE
3465
+ || code == TF_TRANSFORM_INVALID_ARGUMENT)
3466
+ code = tf_transform_set_error(
3467
+ error, TF_TRANSFORM_CORRUPT_PLAN, "embedded recipe is invalid");
3468
+ goto done;
3469
+ }
3470
+ if (!recipe) {
3471
+ code = tf_transform_set_error(
3472
+ error, TF_TRANSFORM_INTERNAL,
3473
+ "embedded recipe parser returned no recipe");
3474
+ goto done;
3475
+ }
3476
+ plan = (tf_transform_plan *)tf_transform_resource_calloc(
3477
+ &ledger, 1, sizeof(*plan), error);
3478
+ if (!plan) {
3479
+ code = ledger.last_code != TF_TRANSFORM_OK
3480
+ ? ledger.last_code : tf_transform_set_error(
3481
+ error, TF_TRANSFORM_ALLOCATION,
3482
+ "imported plan allocation failed");
3483
+ goto done;
3484
+ }
3485
+ atomic_init(&plan->refcount, 1u);
3486
+ plan->recipe = recipe;
3487
+ recipe = NULL;
3488
+ if (!plan->recipe) {
3489
+ code = tf_transform_set_error(
3490
+ error, TF_TRANSFORM_INTERNAL, "recipe parser returned no recipe");
3491
+ goto done;
3492
+ }
3493
+ code = parse_schema_json(
3494
+ required_item(root, "inputSchema"), 0, runtime,
3495
+ &ledger, &plan->input_schema, error);
3496
+ if (code != TF_TRANSFORM_OK) goto done;
3497
+ {
3498
+ const cJSON *output_schema = required_item(root, "outputSchema");
3499
+ const cJSON *field;
3500
+ if (!cJSON_IsArray(output_schema)) {
3501
+ code = tf_transform_set_error(
3502
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3503
+ "plan output schema is not an array");
3504
+ goto done;
3505
+ }
3506
+ cJSON_ArrayForEach(field, output_schema) {
3507
+ if (output_column_count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3508
+ code = tf_transform_poll_cancel(runtime, error);
3509
+ if (code != TF_TRANSFORM_OK) goto done;
3510
+ }
3511
+ if (output_column_count == SIZE_MAX) {
3512
+ code = tf_transform_set_error(
3513
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3514
+ "plan output schema column count overflows");
3515
+ goto done;
3516
+ }
3517
+ ++output_column_count;
3518
+ }
3519
+ }
3520
+ if (plan->input_schema.field_count != plan->recipe->column_count
3521
+ || output_column_count == 0
3522
+ || (uint64_t)output_column_count
3523
+ > plan->recipe->max_output_columns) {
3524
+ code = tf_transform_set_error(
3525
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3526
+ "plan recipe and schema column counts disagree");
3527
+ goto done;
3528
+ }
3529
+ for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
3530
+ const tf_transform_recipe_column *recipe_column = &plan->recipe->columns[i];
3531
+ const tf_transform_schema_field_owned *input = &plan->input_schema.fields[i];
3532
+ int source_equal = 0;
3533
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3534
+ code = tf_transform_poll_cancel(runtime, error);
3535
+ if (code != TF_TRANSFORM_OK) goto done;
3536
+ }
3537
+ code = runtime_span_equal(
3538
+ recipe_column->source_id, recipe_column->source_id_len,
3539
+ input->id, input->id_len, runtime, &source_equal, error);
3540
+ if (code != TF_TRANSFORM_OK) goto done;
3541
+ if (!source_equal
3542
+ || (recipe_column->categorical_fixed_count
3543
+ && recipe_column->categorical_fixed_dtype != input->dtype)
3544
+ || (recipe_column->kind != TF_TRANSFORM_KIND_CATEGORICAL
3545
+ && recipe_column->impute == TF_TRANSFORM_IMPUTE_CONSTANT
3546
+ && recipe_column->constant_dtype != input->dtype)) {
3547
+ code = tf_transform_set_error(
3548
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3549
+ "plan recipe and schema fields disagree");
3550
+ goto done;
3551
+ }
3552
+ }
3553
+ {
3554
+ int valid_fingerprint = 0;
3555
+ code = lowercase_sha256_string_runtime(
3556
+ required_item(root, "recipeSha256"), runtime,
3557
+ &valid_fingerprint, error);
3558
+ if (code != TF_TRANSFORM_OK) goto done;
3559
+ if (!valid_fingerprint) {
3560
+ code = tf_transform_set_error(
3561
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan recipe fingerprint is invalid");
3562
+ goto done;
3563
+ }
3564
+ }
3565
+ code = imported_recipe_fingerprint(
3566
+ root, runtime, &ledger, expected_fingerprint, error);
3567
+ if (code != TF_TRANSFORM_OK) goto done;
3568
+ if (strcmp(required_item(root, "recipeSha256")->valuestring,
3569
+ expected_fingerprint) != 0) {
3570
+ code = tf_transform_set_error(
3571
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan recipe fingerprint mismatches");
3572
+ goto done;
3573
+ }
3574
+ steps = required_item(root, "steps");
3575
+ if (!cJSON_IsArray(steps)) {
3576
+ code = tf_transform_set_error(
3577
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan step count is invalid");
3578
+ goto done;
3579
+ }
3580
+ {
3581
+ const cJSON *step;
3582
+ cJSON_ArrayForEach(step, steps) {
3583
+ if (step_count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3584
+ code = tf_transform_poll_cancel(runtime, error);
3585
+ if (code != TF_TRANSFORM_OK) goto done;
3586
+ }
3587
+ if (step_count == SIZE_MAX) {
3588
+ code = tf_transform_set_error(
3589
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3590
+ "plan step count overflows");
3591
+ goto done;
3592
+ }
3593
+ ++step_count;
3594
+ }
3595
+ }
3596
+ if (step_count != plan->input_schema.field_count) {
3597
+ code = tf_transform_set_error(
3598
+ error, TF_TRANSFORM_CORRUPT_PLAN, "plan step count is invalid");
3599
+ goto done;
3600
+ }
3601
+ {
3602
+ const cJSON *step = steps->child;
3603
+ uint64_t expected_output_columns = 0;
3604
+ uint64_t preflight_total_categories = 0;
3605
+ for (size_t i = 0; i < plan->input_schema.field_count;
3606
+ ++i, step = step->next) {
3607
+ const tf_transform_recipe_column *column
3608
+ = &plan->recipe->columns[i];
3609
+ tf_transform_column_kind resolved_kind;
3610
+ uint64_t width = 1;
3611
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3612
+ code = tf_transform_poll_cancel(runtime, error);
3613
+ if (code != TF_TRANSFORM_OK) goto done;
3614
+ }
3615
+ code = imported_step_kind(
3616
+ step, column, &resolved_kind, error);
3617
+ if (code != TF_TRANSFORM_OK) goto done;
3618
+ if (resolved_kind == TF_TRANSFORM_KIND_CATEGORICAL) {
3619
+ const cJSON *categorical = required_item(step, "categorical");
3620
+ const cJSON *categories = required_item(
3621
+ categorical, "categories");
3622
+ const cJSON *category;
3623
+ size_t count = 0;
3624
+ uint64_t previous_bits = 0;
3625
+ if (!cJSON_IsArray(categories)) {
3626
+ code = tf_transform_set_error(
3627
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3628
+ "categorical plan categories are invalid");
3629
+ goto done;
3630
+ }
3631
+ cJSON_ArrayForEach(category, categories) {
3632
+ uint64_t bits;
3633
+ if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3634
+ code = tf_transform_poll_cancel(runtime, error);
3635
+ if (code != TF_TRANSFORM_OK) goto done;
3636
+ }
3637
+ if (!tagged_category_bits(
3638
+ category, plan->input_schema.fields[i].dtype, &bits)
3639
+ || (column->kind == TF_TRANSFORM_KIND_INFER
3640
+ && !category_bits_are_integer(
3641
+ bits, plan->input_schema.fields[i].dtype))
3642
+ || (count != 0 && tf_transform_category_compare(
3643
+ previous_bits, bits,
3644
+ plan->input_schema.fields[i].dtype) >= 0)) {
3645
+ code = tf_transform_set_error(
3646
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3647
+ "categorical plan state is inconsistent");
3648
+ goto done;
3649
+ }
3650
+ previous_bits = bits;
3651
+ if (count == SIZE_MAX) {
3652
+ code = tf_transform_set_error(
3653
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3654
+ "categorical plan width overflows");
3655
+ goto done;
3656
+ }
3657
+ ++count;
3658
+ }
3659
+ if ((count == 0
3660
+ && !(column->kind == TF_TRANSFORM_KIND_CATEGORICAL
3661
+ && column->categorical_impute
3662
+ == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
3663
+ && column->categorical_encode
3664
+ == TF_TRANSFORM_ENCODE_NONE))
3665
+ || (column->kind == TF_TRANSFORM_KIND_INFER
3666
+ && (count < 2
3667
+ || (uint64_t)count
3668
+ > column->infer_max_categories))) {
3669
+ code = tf_transform_set_error(
3670
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3671
+ "inferred categorical plan cardinality is inconsistent");
3672
+ goto done;
3673
+ }
3674
+ if ((uint64_t)count
3675
+ > runtime->limits.max_categories_per_column
3676
+ || (uint64_t)count
3677
+ > runtime->limits.max_total_categories
3678
+ || preflight_total_categories
3679
+ > runtime->limits.max_total_categories - (uint64_t)count) {
3680
+ code = tf_transform_set_error(
3681
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3682
+ "categorical plan state exceeds host limits");
3683
+ goto done;
3684
+ }
3685
+ preflight_total_categories += (uint64_t)count;
3686
+ if (column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT) {
3687
+ width = (uint64_t)count;
3688
+ if (column->categorical_unknown
3689
+ == TF_TRANSFORM_UNKNOWN_OTHER) {
3690
+ if (width == UINT64_MAX) {
3691
+ code = tf_transform_set_error(
3692
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3693
+ "one-hot plan width overflows");
3694
+ goto done;
3695
+ }
3696
+ ++width;
3697
+ }
3698
+ }
3699
+ }
3700
+ if (width > UINT64_MAX - expected_output_columns) {
3701
+ code = tf_transform_set_error(
3702
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3703
+ "plan output width overflows");
3704
+ goto done;
3705
+ }
3706
+ expected_output_columns += width;
3707
+ }
3708
+ if (expected_output_columns > plan->recipe->max_output_columns
3709
+ || expected_output_columns != (uint64_t)output_column_count) {
3710
+ code = tf_transform_set_error(
3711
+ error, TF_TRANSFORM_CORRUPT_PLAN,
3712
+ "learned plan output width is inconsistent");
3713
+ goto done;
3714
+ }
3715
+ for (size_t i = 0; i < plan->recipe->column_count; ++i) {
3716
+ if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
3717
+ && (plan->recipe->columns[i].infer_max_categories
3718
+ > runtime->limits.max_categories_per_column
3719
+ || plan->recipe->columns[i].infer_max_categories
3720
+ > runtime->limits.max_total_categories)) {
3721
+ code = tf_transform_set_error(
3722
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3723
+ "inferred category bound exceeds host limits");
3724
+ goto done;
3725
+ }
3726
+ }
3727
+ if (expected_output_columns > runtime->limits.max_output_columns) {
3728
+ code = tf_transform_set_error(
3729
+ error, TF_TRANSFORM_RESOURCE_LIMIT,
3730
+ "learned plan output width exceeds host limits");
3731
+ goto done;
3732
+ }
3733
+ }
3734
+ if (plan->input_schema.field_count > SIZE_MAX / sizeof(*plan->states)
3735
+ || (uint64_t)plan->input_schema.field_count * sizeof(*plan->states)
3736
+ > runtime->limits.max_allocation_bytes) {
3737
+ code = tf_transform_set_error(
3738
+ error, TF_TRANSFORM_RESOURCE_LIMIT, "imported plan state exceeds limits");
3739
+ goto done;
3740
+ }
3741
+ plan->states = (tf_transform_column_state *)tf_transform_resource_calloc(
3742
+ &ledger, plan->input_schema.field_count, sizeof(*plan->states), error);
3743
+ if (!plan->states) {
3744
+ code = ledger.last_code != TF_TRANSFORM_OK
3745
+ ? ledger.last_code : tf_transform_set_error(
3746
+ error, TF_TRANSFORM_ALLOCATION,
3747
+ "imported plan state allocation failed");
3748
+ goto done;
3749
+ }
3750
+ {
3751
+ const cJSON *step = steps->child;
3752
+ for (size_t i = 0; i < plan->input_schema.field_count;
3753
+ ++i, step = step->next) {
3754
+ tf_transform_column_kind resolved_kind;
3755
+ if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
3756
+ code = tf_transform_poll_cancel(runtime, error);
3757
+ if (code != TF_TRANSFORM_OK) goto done;
3758
+ }
3759
+ code = imported_step_kind(
3760
+ step, &plan->recipe->columns[i], &resolved_kind, error);
3761
+ if (code != TF_TRANSFORM_OK) goto done;
3762
+ plan->states[i].kind = resolved_kind;
3763
+ if (resolved_kind == TF_TRANSFORM_KIND_CATEGORICAL) {
3764
+ code = parse_categorical_step(
3765
+ step, &plan->recipe->columns[i],
3766
+ plan->input_schema.fields[i].dtype,
3767
+ runtime, &ledger, &total_categories,
3768
+ &plan->states[i].value.categorical, error);
3769
+ } else {
3770
+ code = parse_numeric_step(
3771
+ step, &plan->recipe->columns[i], runtime,
3772
+ &plan->states[i].value.numeric, error);
3773
+ }
3774
+ if (code != TF_TRANSFORM_OK) goto done;
3775
+ }
3776
+ }
3777
+ code = parse_schema_json(
3778
+ required_item(root, "outputSchema"), 1, runtime,
3779
+ &ledger, &plan->output_schema, error);
3780
+ if (code != TF_TRANSFORM_OK) goto done;
3781
+ code = tf_transform_output_schema_validate_contract(
3782
+ &plan->input_schema, plan->recipe, NULL, plan->states,
3783
+ &plan->output_schema, runtime, &ledger,
3784
+ TF_TRANSFORM_CORRUPT_PLAN, error);
3785
+ if (code != TF_TRANSFORM_OK) goto done;
3786
+ code = tf_transform_json_print_canonical_runtime_ledger(
3787
+ root, runtime, &ledger, canonical_out, canonical_len_out, error);
3788
+ if (code != TF_TRANSFORM_OK) goto done;
3789
+ code = tf_transform_poll_cancel(runtime, error);
3790
+ if (code != TF_TRANSFORM_OK) goto done;
3791
+ plan->import_allocation_count = ledger.allocation_count;
3792
+ plan->import_peak_resident_bytes = ledger.peak_resident_bytes;
3793
+ code = TF_TRANSFORM_OK;
3794
+ done:
3795
+ tf_transform_recipe_release(recipe);
3796
+ {
3797
+ int cleanup_cancelled = delete_json_runtime(root, runtime);
3798
+ tf_transform_resource_release(&ledger, ast_resident_bytes);
3799
+ if (code == TF_TRANSFORM_OK && cleanup_cancelled)
3800
+ code = tf_transform_set_error(
3801
+ error, TF_TRANSFORM_CANCELLED,
3802
+ "prepared transform cancelled during JSON cleanup");
3803
+ }
3804
+ if (code == TF_TRANSFORM_OK) {
3805
+ *out = plan;
3806
+ plan = NULL;
3807
+ }
3808
+ tf_transform_plan_release(plan);
3809
+ if (code != TF_TRANSFORM_OK)
3810
+ tf_transform_bytes_free(canonical_out, canonical_len_out);
3811
+ return code;
3812
+ }