tranfi 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/NOTICE +8 -0
- package/README.md +272 -40
- package/app/assets/{index-pDFMluyz.js → index-BIAIKnrp.js} +1 -1
- package/app/index.html +1 -1
- package/binding.gyp +55 -3
- package/csrc/arena.c +7 -5
- package/csrc/batch.c +818 -71
- package/csrc/buffer.c +84 -8
- package/csrc/cJSON.c +262 -19
- package/csrc/cJSON.h +17 -1
- package/csrc/codec_csv.c +1074 -181
- package/csrc/codec_jsonl.c +830 -118
- package/csrc/codec_table.c +108 -78
- package/csrc/codec_text.c +286 -68
- package/csrc/compiler.c +31 -3
- package/csrc/config.h +21 -0
- package/csrc/dsl.c +4722 -485
- package/csrc/expr.c +363 -55
- package/csrc/expr.h +2 -0
- package/csrc/internal.h +316 -27
- package/csrc/ir.c +65 -18
- package/csrc/ir.h +41 -0
- package/csrc/ir_schema.c +20 -5
- package/csrc/ir_serialize.c +68 -6
- package/csrc/ir_sql.c +796 -185
- package/csrc/ir_validate.c +462 -6
- package/csrc/json_path.c +210 -0
- package/csrc/main.c +879 -30
- package/csrc/memory_estimate.c +477 -0
- package/csrc/op_acf.c +171 -21
- package/csrc/op_across.c +477 -0
- package/csrc/op_anomaly.c +167 -32
- package/csrc/op_assert.c +761 -0
- package/csrc/op_bin.c +168 -29
- package/csrc/op_cast.c +383 -55
- package/csrc/op_clip.c +30 -19
- package/csrc/op_date_trunc.c +208 -34
- package/csrc/op_datetime.c +259 -77
- package/csrc/op_derive.c +65 -97
- package/csrc/op_diff.c +146 -30
- package/csrc/op_ewma.c +149 -30
- package/csrc/op_explode.c +124 -26
- package/csrc/op_fill_down.c +125 -53
- package/csrc/op_fill_null.c +176 -31
- package/csrc/op_filter.c +89 -40
- package/csrc/op_frequency.c +571 -43
- package/csrc/op_grep.c +36 -18
- package/csrc/op_group_agg.c +1790 -119
- package/csrc/op_hash.c +48 -15
- package/csrc/op_head.c +21 -86
- package/csrc/op_interpolate.c +268 -62
- package/csrc/op_join.c +2700 -182
- package/csrc/op_json_extract.c +227 -0
- package/csrc/op_json_filter.c +384 -0
- package/csrc/op_json_flatten.c +293 -0
- package/csrc/op_json_schema.c +503 -0
- package/csrc/op_label_encode.c +328 -53
- package/csrc/op_lag.c +181 -0
- package/csrc/op_lead.c +141 -89
- package/csrc/op_normalize.c +363 -79
- package/csrc/op_onehot.c +345 -73
- package/csrc/op_pivot.c +1546 -162
- package/csrc/op_quarantine.c +189 -0
- package/csrc/op_registry.c +2062 -166
- package/csrc/op_rename.c +41 -50
- package/csrc/op_replace.c +270 -118
- package/csrc/op_rleid.c +297 -0
- package/csrc/op_rowid.c +559 -0
- package/csrc/op_sample.c +80 -23
- package/csrc/op_schema.c +1341 -0
- package/csrc/op_schema_infer.c +252 -0
- package/csrc/op_select.c +265 -65
- package/csrc/op_set.c +3449 -0
- package/csrc/op_skip.c +30 -87
- package/csrc/op_sort.c +670 -124
- package/csrc/op_source_name.c +120 -0
- package/csrc/op_split.c +65 -28
- package/csrc/op_split_data.c +41 -9
- package/csrc/op_stack.c +178 -222
- package/csrc/op_stats.c +206 -110
- package/csrc/op_step.c +217 -55
- package/csrc/op_tail.c +21 -12
- package/csrc/op_tee.c +338 -0
- package/csrc/op_top.c +260 -53
- package/csrc/op_trim.c +48 -19
- package/csrc/op_unique.c +1193 -150
- package/csrc/op_unpivot.c +100 -66
- package/csrc/op_validate.c +601 -24
- package/csrc/op_window.c +492 -51
- package/csrc/path_policy.c +85 -0
- package/csrc/pipeline.c +872 -99
- package/csrc/recipes.c +3 -1
- package/csrc/report.c +73 -30
- package/csrc/selector.c +1097 -0
- package/csrc/size_utils.c +348 -0
- package/csrc/spill.c +317 -0
- package/csrc/spill.h +21 -0
- package/csrc/tranfi.h +169 -1
- package/csrc/transform.h +209 -0
- package/csrc/transform_api.c +2237 -0
- package/csrc/transform_categorical.c +923 -0
- package/csrc/transform_internal.h +472 -0
- package/csrc/transform_json.c +3812 -0
- package/csrc/transform_numeric.c +1966 -0
- package/csrc/transform_sha256.c +154 -0
- package/csrc/transform_wasm.h +162 -0
- package/csrc/transform_wasm_api.c +1373 -0
- package/csrc/wasm_api.c +70 -9
- package/napi_api.c +219 -11
- package/napi_transform.c +1648 -0
- package/napi_transform.h +8 -0
- package/package.json +27 -11
- package/scripts/install-native.js +76 -0
- package/scripts/prepack.js +64 -0
- package/scripts/sync-csrc.js +23 -0
- package/src/cli.js +8 -11
- package/src/engines/duckdb.js +45 -12
- package/src/index.js +661 -42
- package/src/memory_policy.js +411 -0
- package/src/native.js +1 -5
- package/src/pipeline.js +454 -31
- package/src/recipe_json.js +80 -0
- package/src/server.js +10 -8
- package/src/transform.js +403 -0
- package/src/transform_error.js +10 -0
- package/src/wasm.js +6 -4
- package/wasm/index.js +498 -10
- package/wasm/tranfi_core.js +0 -0
- package/wasm/transform.js +1156 -0
- package/wasm/worker.js +786 -0
- package/csrc/plan.c +0 -206
|
@@ -0,0 +1,3812 @@
|
|
|
1
|
+
#include "transform_internal.h"
|
|
2
|
+
|
|
3
|
+
#include <math.h>
|
|
4
|
+
#include <stdio.h>
|
|
5
|
+
#include <stdlib.h>
|
|
6
|
+
#include <string.h>
|
|
7
|
+
|
|
8
|
+
typedef struct tf_json_work {
|
|
9
|
+
const tf_transform_runtime_copy *runtime;
|
|
10
|
+
tf_transform_resource_ledger *ledger;
|
|
11
|
+
tf_transform_error **error;
|
|
12
|
+
size_t bytes_since_poll;
|
|
13
|
+
size_t iterations_since_poll;
|
|
14
|
+
int cancelled;
|
|
15
|
+
} tf_json_work;
|
|
16
|
+
|
|
17
|
+
typedef struct tf_json_scan_metrics {
|
|
18
|
+
uint64_t node_count_upper;
|
|
19
|
+
uint64_t string_count;
|
|
20
|
+
uint64_t number_count;
|
|
21
|
+
uint64_t object_count;
|
|
22
|
+
uint64_t decoded_string_bytes;
|
|
23
|
+
uint64_t max_string_bytes;
|
|
24
|
+
uint64_t max_number_bytes;
|
|
25
|
+
uint64_t resident_bytes_persistent;
|
|
26
|
+
uint64_t resident_bytes_upper;
|
|
27
|
+
uint64_t allocation_count_upper;
|
|
28
|
+
} tf_json_scan_metrics;
|
|
29
|
+
|
|
30
|
+
static tf_transform_code json_work_progress(
|
|
31
|
+
tf_json_work *work, size_t bytes, size_t iterations, int force) {
|
|
32
|
+
tf_transform_code code;
|
|
33
|
+
if (!work || !work->runtime) return TF_TRANSFORM_OK;
|
|
34
|
+
if (bytes > SIZE_MAX - work->bytes_since_poll
|
|
35
|
+
|| iterations > SIZE_MAX - work->iterations_since_poll)
|
|
36
|
+
return tf_transform_set_error(
|
|
37
|
+
work->error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
38
|
+
"JSON progress counter overflows");
|
|
39
|
+
work->bytes_since_poll += bytes;
|
|
40
|
+
work->iterations_since_poll += iterations;
|
|
41
|
+
if (!force
|
|
42
|
+
&& work->bytes_since_poll < TF_TRANSFORM_CANCEL_BYTES_V1
|
|
43
|
+
&& work->iterations_since_poll < TF_TRANSFORM_CANCEL_ITERS_V1)
|
|
44
|
+
return TF_TRANSFORM_OK;
|
|
45
|
+
code = tf_transform_poll_cancel(work->runtime, work->error);
|
|
46
|
+
if (code != TF_TRANSFORM_OK) {
|
|
47
|
+
work->cancelled = 1;
|
|
48
|
+
return code;
|
|
49
|
+
}
|
|
50
|
+
work->bytes_since_poll = 0;
|
|
51
|
+
work->iterations_since_poll = 0;
|
|
52
|
+
return TF_TRANSFORM_OK;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
static cJSON_bool json_parser_poll(void *user_data) {
|
|
56
|
+
tf_json_work *work = (tf_json_work *)user_data;
|
|
57
|
+
if (!work || !work->runtime || !work->runtime->cancel) return 1;
|
|
58
|
+
if (work->runtime->cancel(work->runtime->cancel_user)) {
|
|
59
|
+
work->cancelled = 1;
|
|
60
|
+
return 0;
|
|
61
|
+
}
|
|
62
|
+
return 1;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
static tf_transform_code json_work_allocation(
|
|
66
|
+
tf_json_work *work, uint64_t bytes, tf_transform_error **error) {
|
|
67
|
+
if (!work || !work->ledger) return tf_transform_set_error(
|
|
68
|
+
error, TF_TRANSFORM_INTERNAL, "JSON resource ledger is missing");
|
|
69
|
+
return tf_transform_resource_reserve(work->ledger, bytes, error);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
static void json_work_release(tf_json_work *work, uint64_t bytes) {
|
|
73
|
+
if (work) tf_transform_resource_release(work->ledger, bytes);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
static tf_transform_code buffer_reserve(
|
|
77
|
+
tf_transform_buffer *buffer, size_t extra,
|
|
78
|
+
const tf_transform_limits_v1 *limits, tf_json_work *work,
|
|
79
|
+
tf_transform_error **error) {
|
|
80
|
+
size_t needed;
|
|
81
|
+
size_t capacity;
|
|
82
|
+
uint8_t *data;
|
|
83
|
+
if (extra > SIZE_MAX - buffer->len)
|
|
84
|
+
return tf_transform_set_error(
|
|
85
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON size overflow");
|
|
86
|
+
needed = buffer->len + extra;
|
|
87
|
+
if (needed > limits->max_plan_bytes || needed > limits->max_allocation_bytes)
|
|
88
|
+
return tf_transform_set_error(
|
|
89
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON exceeds byte limit");
|
|
90
|
+
if (buffer->count_only || needed <= buffer->cap) return TF_TRANSFORM_OK;
|
|
91
|
+
capacity = needed;
|
|
92
|
+
if (buffer->data || capacity != buffer->cap)
|
|
93
|
+
return tf_transform_set_error(
|
|
94
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
95
|
+
"canonical JSON fixed buffer is too small");
|
|
96
|
+
(void)work;
|
|
97
|
+
(void)data;
|
|
98
|
+
return TF_TRANSFORM_OK;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
static tf_transform_code buffer_write(
|
|
102
|
+
tf_transform_buffer *buffer, const void *data, size_t len,
|
|
103
|
+
const tf_transform_limits_v1 *limits, tf_json_work *work,
|
|
104
|
+
tf_transform_error **error) {
|
|
105
|
+
tf_transform_code code = buffer_reserve(
|
|
106
|
+
buffer, len, limits, work, error);
|
|
107
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
108
|
+
if (len != 0 && (!buffer->count_only && !buffer->data))
|
|
109
|
+
return tf_transform_set_error(
|
|
110
|
+
error, TF_TRANSFORM_INTERNAL, "canonical JSON buffer invariant failed");
|
|
111
|
+
if (len != 0 && !data)
|
|
112
|
+
return tf_transform_set_error(
|
|
113
|
+
error, TF_TRANSFORM_INTERNAL, "canonical JSON source is null");
|
|
114
|
+
if (len != 0 && !buffer->count_only)
|
|
115
|
+
memcpy(buffer->data + buffer->len, data, len);
|
|
116
|
+
buffer->len += len;
|
|
117
|
+
return json_work_progress(work, len, 0, 0);
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
static tf_transform_code write_json_string(
|
|
121
|
+
tf_transform_buffer *buffer, const char *text,
|
|
122
|
+
const tf_transform_limits_v1 *limits, tf_json_work *work,
|
|
123
|
+
tf_transform_error **error) {
|
|
124
|
+
static const char hex[] = "0123456789abcdef";
|
|
125
|
+
tf_transform_code code;
|
|
126
|
+
size_t len = 0;
|
|
127
|
+
if (!text) return tf_transform_set_error(
|
|
128
|
+
error, TF_TRANSFORM_INTERNAL, "JSON string is null");
|
|
129
|
+
while (len <= limits->max_string_bytes && text[len] != '\0') {
|
|
130
|
+
code = json_work_progress(work, 1, 0, 0);
|
|
131
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
132
|
+
++len;
|
|
133
|
+
}
|
|
134
|
+
if (len > limits->max_string_bytes) return tf_transform_set_error(
|
|
135
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON string exceeds limit");
|
|
136
|
+
code = buffer_write(buffer, "\"", 1, limits, work, error);
|
|
137
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
138
|
+
for (size_t i = 0; i < len; ++i) {
|
|
139
|
+
unsigned char byte = (unsigned char)text[i];
|
|
140
|
+
if (byte == '"' || byte == '\\') {
|
|
141
|
+
char escaped[2] = {'\\', (char)byte};
|
|
142
|
+
code = buffer_write(
|
|
143
|
+
buffer, escaped, sizeof(escaped), limits, work, error);
|
|
144
|
+
} else if (byte < 0x20) {
|
|
145
|
+
char escaped[6] = {'\\', 'u', '0', '0', hex[byte >> 4], hex[byte & 15]};
|
|
146
|
+
code = buffer_write(
|
|
147
|
+
buffer, escaped, sizeof(escaped), limits, work, error);
|
|
148
|
+
} else {
|
|
149
|
+
code = buffer_write(buffer, &text[i], 1, limits, work, error);
|
|
150
|
+
}
|
|
151
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
152
|
+
}
|
|
153
|
+
return buffer_write(buffer, "\"", 1, limits, work, error);
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
static tf_transform_code compare_object_keys(
|
|
157
|
+
const cJSON *left, const cJSON *right,
|
|
158
|
+
const tf_transform_limits_v1 *limits, tf_json_work *work,
|
|
159
|
+
int *result, tf_transform_error **error) {
|
|
160
|
+
const unsigned char *a;
|
|
161
|
+
const unsigned char *b;
|
|
162
|
+
size_t compared = 0;
|
|
163
|
+
if (!left || !right || !left->string || !right->string || !result)
|
|
164
|
+
return tf_transform_set_error(
|
|
165
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
166
|
+
"canonical JSON object key is invalid");
|
|
167
|
+
a = (const unsigned char *)left->string;
|
|
168
|
+
b = (const unsigned char *)right->string;
|
|
169
|
+
while (compared <= limits->max_string_bytes && *a && *b && *a == *b) {
|
|
170
|
+
tf_transform_code code = json_work_progress(work, 1, 0, 0);
|
|
171
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
172
|
+
++a;
|
|
173
|
+
++b;
|
|
174
|
+
++compared;
|
|
175
|
+
}
|
|
176
|
+
if (compared > limits->max_string_bytes)
|
|
177
|
+
return tf_transform_set_error(
|
|
178
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
179
|
+
"canonical JSON object key exceeds limit");
|
|
180
|
+
*result = *a < *b ? -1 : (*a > *b ? 1 : 0);
|
|
181
|
+
return TF_TRANSFORM_OK;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
static tf_transform_code sort_object_items(
|
|
185
|
+
const cJSON **items, size_t count, tf_json_work *work,
|
|
186
|
+
tf_transform_error **error) {
|
|
187
|
+
size_t start;
|
|
188
|
+
size_t end;
|
|
189
|
+
(void)error;
|
|
190
|
+
if (count < 2) return TF_TRANSFORM_OK;
|
|
191
|
+
start = count / 2;
|
|
192
|
+
end = count;
|
|
193
|
+
for (;;) {
|
|
194
|
+
size_t root;
|
|
195
|
+
const cJSON *saved;
|
|
196
|
+
tf_transform_code code;
|
|
197
|
+
if (start != 0) {
|
|
198
|
+
--start;
|
|
199
|
+
root = start;
|
|
200
|
+
saved = items[root];
|
|
201
|
+
} else {
|
|
202
|
+
--end;
|
|
203
|
+
if (end == 0) break;
|
|
204
|
+
saved = items[end];
|
|
205
|
+
items[end] = items[0];
|
|
206
|
+
root = 0;
|
|
207
|
+
}
|
|
208
|
+
while (root <= (end - 1) / 2 && end > 1) {
|
|
209
|
+
size_t child = root * 2 + 1;
|
|
210
|
+
if (child + 1 < end) {
|
|
211
|
+
int compared;
|
|
212
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
213
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
214
|
+
code = compare_object_keys(
|
|
215
|
+
items[child], items[child + 1],
|
|
216
|
+
&work->runtime->limits, work, &compared, error);
|
|
217
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
218
|
+
if (compared < 0)
|
|
219
|
+
++child;
|
|
220
|
+
}
|
|
221
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
222
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
223
|
+
{
|
|
224
|
+
int compared;
|
|
225
|
+
code = compare_object_keys(
|
|
226
|
+
saved, items[child], &work->runtime->limits,
|
|
227
|
+
work, &compared, error);
|
|
228
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
229
|
+
if (compared >= 0) break;
|
|
230
|
+
}
|
|
231
|
+
items[root] = items[child];
|
|
232
|
+
root = child;
|
|
233
|
+
}
|
|
234
|
+
items[root] = saved;
|
|
235
|
+
}
|
|
236
|
+
return TF_TRANSFORM_OK;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
static tf_transform_code write_canonical_value(
|
|
240
|
+
const cJSON *value, tf_transform_buffer *buffer,
|
|
241
|
+
const tf_transform_limits_v1 *limits, size_t depth,
|
|
242
|
+
tf_json_work *work, tf_transform_error **error) {
|
|
243
|
+
tf_transform_code code;
|
|
244
|
+
if (!value || depth > limits->max_json_depth)
|
|
245
|
+
return tf_transform_set_error(
|
|
246
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical JSON depth exceeds limit");
|
|
247
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
248
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
249
|
+
if (cJSON_IsNull(value))
|
|
250
|
+
return buffer_write(buffer, "null", 4, limits, work, error);
|
|
251
|
+
if (cJSON_IsTrue(value))
|
|
252
|
+
return buffer_write(buffer, "true", 4, limits, work, error);
|
|
253
|
+
if (cJSON_IsFalse(value))
|
|
254
|
+
return buffer_write(buffer, "false", 5, limits, work, error);
|
|
255
|
+
if (cJSON_IsString(value))
|
|
256
|
+
return write_json_string(
|
|
257
|
+
buffer, value->valuestring, limits, work, error);
|
|
258
|
+
if (cJSON_IsNumber(value)) {
|
|
259
|
+
char number[32];
|
|
260
|
+
int count;
|
|
261
|
+
double integer;
|
|
262
|
+
if (!isfinite(value->valuedouble)
|
|
263
|
+
|| modf(value->valuedouble, &integer) != 0.0
|
|
264
|
+
|| integer < -9007199254740991.0
|
|
265
|
+
|| integer > 9007199254740991.0)
|
|
266
|
+
return tf_transform_set_error(
|
|
267
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
268
|
+
"canonical JSON number is not a safe integer");
|
|
269
|
+
if (integer == 0.0) integer = 0.0;
|
|
270
|
+
count = snprintf(number, sizeof(number), "%.0f", integer);
|
|
271
|
+
if (count <= 0 || (size_t)count >= sizeof(number))
|
|
272
|
+
return tf_transform_set_error(
|
|
273
|
+
error, TF_TRANSFORM_INTERNAL, "canonical integer formatting failed");
|
|
274
|
+
return buffer_write(
|
|
275
|
+
buffer, number, (size_t)count, limits, work, error);
|
|
276
|
+
}
|
|
277
|
+
if (cJSON_IsArray(value)) {
|
|
278
|
+
const cJSON *child;
|
|
279
|
+
int first = 1;
|
|
280
|
+
code = buffer_write(buffer, "[", 1, limits, work, error);
|
|
281
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
282
|
+
cJSON_ArrayForEach(child, value) {
|
|
283
|
+
if (!first) {
|
|
284
|
+
code = buffer_write(buffer, ",", 1, limits, work, error);
|
|
285
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
286
|
+
}
|
|
287
|
+
first = 0;
|
|
288
|
+
code = write_canonical_value(
|
|
289
|
+
child, buffer, limits, depth + 1, work, error);
|
|
290
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
291
|
+
}
|
|
292
|
+
return buffer_write(buffer, "]", 1, limits, work, error);
|
|
293
|
+
}
|
|
294
|
+
if (cJSON_IsObject(value)) {
|
|
295
|
+
const cJSON *child;
|
|
296
|
+
const cJSON *count_child = value->child;
|
|
297
|
+
const cJSON **items;
|
|
298
|
+
size_t count = 0;
|
|
299
|
+
size_t index = 0;
|
|
300
|
+
cJSON_ArrayForEach(child, value) {
|
|
301
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
302
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
303
|
+
++count;
|
|
304
|
+
}
|
|
305
|
+
if (count > limits->max_object_keys
|
|
306
|
+
|| count > SIZE_MAX / sizeof(items[0]))
|
|
307
|
+
return tf_transform_set_error(
|
|
308
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "canonical object exceeds key limit");
|
|
309
|
+
if (count > limits->max_allocation_bytes / sizeof(items[0]))
|
|
310
|
+
return tf_transform_set_error(
|
|
311
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
312
|
+
"canonical key array exceeds allocation limit");
|
|
313
|
+
items = NULL;
|
|
314
|
+
if (!buffer->count_only && count != 0) {
|
|
315
|
+
uint64_t allocation_bytes = (uint64_t)count * sizeof(items[0]);
|
|
316
|
+
code = json_work_allocation(work, allocation_bytes, error);
|
|
317
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
318
|
+
items = (const cJSON **)malloc((size_t)allocation_bytes);
|
|
319
|
+
}
|
|
320
|
+
if (!buffer->count_only && count && !items) {
|
|
321
|
+
json_work_release(work, (uint64_t)count * sizeof(items[0]));
|
|
322
|
+
return tf_transform_set_error(
|
|
323
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
324
|
+
"canonical key allocation failed");
|
|
325
|
+
}
|
|
326
|
+
if (!buffer->count_only) {
|
|
327
|
+
cJSON_ArrayForEach(child, value) items[index++] = child;
|
|
328
|
+
code = sort_object_items(items, count, work, error);
|
|
329
|
+
if (code != TF_TRANSFORM_OK) goto object_done;
|
|
330
|
+
for (size_t i = 1; i < count; ++i) {
|
|
331
|
+
int compared;
|
|
332
|
+
code = compare_object_keys(
|
|
333
|
+
items[i - 1], items[i], limits,
|
|
334
|
+
work, &compared, error);
|
|
335
|
+
if (code != TF_TRANSFORM_OK) goto object_done;
|
|
336
|
+
if (compared == 0) {
|
|
337
|
+
code = tf_transform_set_error(
|
|
338
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
339
|
+
"duplicate JSON object key");
|
|
340
|
+
goto object_done;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
code = buffer_write(buffer, "{", 1, limits, work, error);
|
|
345
|
+
for (size_t i = 0; code == TF_TRANSFORM_OK && i < count; ++i) {
|
|
346
|
+
const cJSON *item = buffer->count_only ? count_child : items[i];
|
|
347
|
+
if (!item) {
|
|
348
|
+
code = tf_transform_set_error(
|
|
349
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
350
|
+
"canonical object traversal failed");
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
if (i != 0)
|
|
354
|
+
code = buffer_write(buffer, ",", 1, limits, work, error);
|
|
355
|
+
if (code == TF_TRANSFORM_OK) code = write_json_string(
|
|
356
|
+
buffer, item->string, limits, work, error);
|
|
357
|
+
if (code == TF_TRANSFORM_OK)
|
|
358
|
+
code = buffer_write(buffer, ":", 1, limits, work, error);
|
|
359
|
+
if (code == TF_TRANSFORM_OK)
|
|
360
|
+
code = write_canonical_value(
|
|
361
|
+
item, buffer, limits, depth + 1, work, error);
|
|
362
|
+
if (buffer->count_only) count_child = item->next;
|
|
363
|
+
}
|
|
364
|
+
object_done:
|
|
365
|
+
free(items);
|
|
366
|
+
if (!buffer->count_only && count != 0)
|
|
367
|
+
json_work_release(work, (uint64_t)count * sizeof(items[0]));
|
|
368
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
369
|
+
return buffer_write(buffer, "}", 1, limits, work, error);
|
|
370
|
+
}
|
|
371
|
+
return tf_transform_set_error(
|
|
372
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "unsupported canonical JSON value");
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
static tf_transform_code json_print_canonical_common(
|
|
376
|
+
const cJSON *value, const tf_transform_runtime_copy *runtime,
|
|
377
|
+
tf_transform_resource_ledger *shared_ledger,
|
|
378
|
+
uint8_t **out, size_t *out_len, tf_transform_error **error) {
|
|
379
|
+
tf_transform_buffer count_buffer = {0};
|
|
380
|
+
tf_transform_buffer write_buffer = {0};
|
|
381
|
+
tf_json_work work = {0};
|
|
382
|
+
tf_transform_resource_ledger local_ledger;
|
|
383
|
+
tf_transform_code code;
|
|
384
|
+
if (out) *out = NULL;
|
|
385
|
+
if (out_len) *out_len = 0;
|
|
386
|
+
tf_transform_clear_error(error);
|
|
387
|
+
if (!value || !runtime || !out || !out_len)
|
|
388
|
+
return tf_transform_set_error(
|
|
389
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "canonical JSON argument is null");
|
|
390
|
+
work.runtime = runtime;
|
|
391
|
+
if (shared_ledger) work.ledger = shared_ledger;
|
|
392
|
+
else {
|
|
393
|
+
tf_transform_resource_ledger_init(&local_ledger, runtime);
|
|
394
|
+
work.ledger = &local_ledger;
|
|
395
|
+
}
|
|
396
|
+
work.error = error;
|
|
397
|
+
code = json_work_progress(&work, 0, 0, 1);
|
|
398
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
399
|
+
count_buffer.count_only = 1;
|
|
400
|
+
code = write_canonical_value(
|
|
401
|
+
value, &count_buffer, &runtime->limits, 1, &work, error);
|
|
402
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
403
|
+
if ((uint64_t)count_buffer.len > runtime->limits.max_plan_bytes
|
|
404
|
+
|| (uint64_t)count_buffer.len > runtime->limits.max_allocation_bytes)
|
|
405
|
+
return tf_transform_set_error(
|
|
406
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
407
|
+
"canonical JSON output exceeds limits");
|
|
408
|
+
if (count_buffer.len != 0) {
|
|
409
|
+
code = json_work_allocation(&work, (uint64_t)count_buffer.len, error);
|
|
410
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
411
|
+
write_buffer.data = (uint8_t *)malloc(count_buffer.len);
|
|
412
|
+
if (!write_buffer.data) {
|
|
413
|
+
json_work_release(&work, (uint64_t)count_buffer.len);
|
|
414
|
+
return tf_transform_set_error(
|
|
415
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
416
|
+
"canonical JSON allocation failed");
|
|
417
|
+
}
|
|
418
|
+
write_buffer.cap = count_buffer.len;
|
|
419
|
+
}
|
|
420
|
+
code = write_canonical_value(
|
|
421
|
+
value, &write_buffer, &runtime->limits, 1, &work, error);
|
|
422
|
+
if (code != TF_TRANSFORM_OK) {
|
|
423
|
+
free(write_buffer.data);
|
|
424
|
+
json_work_release(&work, (uint64_t)count_buffer.len);
|
|
425
|
+
return code;
|
|
426
|
+
}
|
|
427
|
+
if (write_buffer.len != count_buffer.len) {
|
|
428
|
+
free(write_buffer.data);
|
|
429
|
+
json_work_release(&work, (uint64_t)count_buffer.len);
|
|
430
|
+
return tf_transform_set_error(
|
|
431
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
432
|
+
"canonical JSON sizing pass drifted");
|
|
433
|
+
}
|
|
434
|
+
code = json_work_progress(&work, 0, 0, 1);
|
|
435
|
+
if (code != TF_TRANSFORM_OK) {
|
|
436
|
+
free(write_buffer.data);
|
|
437
|
+
json_work_release(&work, (uint64_t)count_buffer.len);
|
|
438
|
+
return code;
|
|
439
|
+
}
|
|
440
|
+
*out = write_buffer.data;
|
|
441
|
+
*out_len = write_buffer.len;
|
|
442
|
+
return TF_TRANSFORM_OK;
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
tf_transform_code tf_transform_json_print_canonical_runtime(
|
|
446
|
+
const cJSON *value, const tf_transform_runtime_copy *runtime,
|
|
447
|
+
uint8_t **out, size_t *out_len, tf_transform_error **error) {
|
|
448
|
+
return json_print_canonical_common(
|
|
449
|
+
value, runtime, NULL, out, out_len, error);
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
tf_transform_code tf_transform_json_print_canonical_runtime_ledger(
|
|
453
|
+
const cJSON *value, const tf_transform_runtime_copy *runtime,
|
|
454
|
+
tf_transform_resource_ledger *ledger,
|
|
455
|
+
uint8_t **out, size_t *out_len, tf_transform_error **error) {
|
|
456
|
+
if (!ledger || ledger->runtime != runtime) {
|
|
457
|
+
if (out) *out = NULL;
|
|
458
|
+
if (out_len) *out_len = 0;
|
|
459
|
+
return tf_transform_set_error(
|
|
460
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
461
|
+
"canonical JSON resource ledger is invalid");
|
|
462
|
+
}
|
|
463
|
+
return json_print_canonical_common(
|
|
464
|
+
value, runtime, ledger, out, out_len, error);
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
tf_transform_code tf_transform_json_print_canonical(
|
|
468
|
+
const cJSON *value, const tf_transform_limits_v1 *limits,
|
|
469
|
+
uint8_t **out, size_t *out_len, tf_transform_error **error) {
|
|
470
|
+
tf_transform_runtime_copy runtime;
|
|
471
|
+
if (!limits) {
|
|
472
|
+
if (out) *out = NULL;
|
|
473
|
+
if (out_len) *out_len = 0;
|
|
474
|
+
return tf_transform_set_error(
|
|
475
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
476
|
+
"canonical JSON limits are null");
|
|
477
|
+
}
|
|
478
|
+
memset(&runtime, 0, sizeof(runtime));
|
|
479
|
+
runtime.limits = *limits;
|
|
480
|
+
return json_print_canonical_common(
|
|
481
|
+
value, &runtime, NULL, out, out_len, error);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
static int json_string_token_equals_ascii(
|
|
485
|
+
const uint8_t *token, size_t len, const char *expected) {
|
|
486
|
+
size_t input = 0;
|
|
487
|
+
size_t output = 0;
|
|
488
|
+
size_t expected_len = strlen(expected);
|
|
489
|
+
while (input < len) {
|
|
490
|
+
uint32_t decoded = token[input++];
|
|
491
|
+
if (decoded == '\\') {
|
|
492
|
+
unsigned char escaped;
|
|
493
|
+
if (input >= len) return 0;
|
|
494
|
+
escaped = token[input++];
|
|
495
|
+
if (escaped == 'u') {
|
|
496
|
+
decoded = 0;
|
|
497
|
+
if (len - input < 4) return 0;
|
|
498
|
+
for (size_t i = 0; i < 4; ++i) {
|
|
499
|
+
unsigned char byte = token[input++];
|
|
500
|
+
unsigned int nibble;
|
|
501
|
+
if (byte >= '0' && byte <= '9') nibble = byte - '0';
|
|
502
|
+
else if (byte >= 'a' && byte <= 'f')
|
|
503
|
+
nibble = byte - 'a' + 10u;
|
|
504
|
+
else if (byte >= 'A' && byte <= 'F')
|
|
505
|
+
nibble = byte - 'A' + 10u;
|
|
506
|
+
else return 0;
|
|
507
|
+
decoded = (decoded << 4) | nibble;
|
|
508
|
+
}
|
|
509
|
+
} else if (escaped == '"' || escaped == '\\' || escaped == '/') {
|
|
510
|
+
decoded = escaped;
|
|
511
|
+
} else if (escaped == 'b') decoded = '\b';
|
|
512
|
+
else if (escaped == 'f') decoded = '\f';
|
|
513
|
+
else if (escaped == 'n') decoded = '\n';
|
|
514
|
+
else if (escaped == 'r') decoded = '\r';
|
|
515
|
+
else if (escaped == 't') decoded = '\t';
|
|
516
|
+
else return 0;
|
|
517
|
+
}
|
|
518
|
+
if (decoded > UINT8_MAX || output >= expected_len
|
|
519
|
+
|| (unsigned char)expected[output] != (unsigned char)decoded)
|
|
520
|
+
return 0;
|
|
521
|
+
++output;
|
|
522
|
+
}
|
|
523
|
+
return output == expected_len;
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
static tf_transform_code scan_json_array_value(
|
|
527
|
+
size_t depth, const char *containers,
|
|
528
|
+
unsigned char *array_expects_value,
|
|
529
|
+
const unsigned char *category_arrays,
|
|
530
|
+
uint64_t *category_counts, size_t *array_elements,
|
|
531
|
+
uint64_t *total_categories, const tf_transform_limits_v1 *limits,
|
|
532
|
+
tf_transform_error **error) {
|
|
533
|
+
size_t index;
|
|
534
|
+
if (depth == 0 || containers[depth - 1] != '['
|
|
535
|
+
|| !array_expects_value[depth - 1]) return TF_TRANSFORM_OK;
|
|
536
|
+
index = depth - 1;
|
|
537
|
+
if (*array_elements == SIZE_MAX)
|
|
538
|
+
return tf_transform_set_error(
|
|
539
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
540
|
+
"JSON array element count overflows");
|
|
541
|
+
++*array_elements;
|
|
542
|
+
array_expects_value[index] = 0;
|
|
543
|
+
if (!category_arrays[index]) return TF_TRANSFORM_OK;
|
|
544
|
+
if (category_counts[index] == UINT64_MAX
|
|
545
|
+
|| *total_categories == UINT64_MAX)
|
|
546
|
+
return tf_transform_set_error(
|
|
547
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
548
|
+
"JSON category count overflows");
|
|
549
|
+
++category_counts[index];
|
|
550
|
+
++*total_categories;
|
|
551
|
+
if (category_counts[index] > limits->max_categories_per_column
|
|
552
|
+
|| *total_categories > limits->max_total_categories)
|
|
553
|
+
return tf_transform_set_error(
|
|
554
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
555
|
+
"JSON categories exceed configured limits before parsing");
|
|
556
|
+
return TF_TRANSFORM_OK;
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
static tf_transform_code scan_json_bounds(
|
|
560
|
+
const uint8_t *json, size_t len, const tf_transform_limits_v1 *limits,
|
|
561
|
+
tf_json_work *work, tf_json_scan_metrics *metrics,
|
|
562
|
+
tf_transform_error **error, tf_transform_code malformed_code) {
|
|
563
|
+
size_t depth = 0;
|
|
564
|
+
size_t keys = 0;
|
|
565
|
+
size_t strings = 0;
|
|
566
|
+
size_t current_string = 0;
|
|
567
|
+
size_t array_elements = 0;
|
|
568
|
+
size_t objects = 0;
|
|
569
|
+
size_t string_count = 0;
|
|
570
|
+
size_t number_count = 0;
|
|
571
|
+
size_t max_string = 0;
|
|
572
|
+
size_t current_number = 0;
|
|
573
|
+
size_t max_number = 0;
|
|
574
|
+
size_t last_string_start = 0;
|
|
575
|
+
size_t last_string_end = 0;
|
|
576
|
+
uint64_t total_categories = 0;
|
|
577
|
+
int in_string = 0;
|
|
578
|
+
int escaped = 0;
|
|
579
|
+
int in_number = 0;
|
|
580
|
+
int have_last_string = 0;
|
|
581
|
+
int expect_categories_array = 0;
|
|
582
|
+
char containers[CJSON_NESTING_LIMIT];
|
|
583
|
+
unsigned char array_expects_value[CJSON_NESTING_LIMIT];
|
|
584
|
+
unsigned char category_arrays[CJSON_NESTING_LIMIT];
|
|
585
|
+
uint64_t category_counts[CJSON_NESTING_LIMIT];
|
|
586
|
+
memset(category_arrays, 0, sizeof(category_arrays));
|
|
587
|
+
memset(category_counts, 0, sizeof(category_counts));
|
|
588
|
+
memset(metrics, 0, sizeof(*metrics));
|
|
589
|
+
for (size_t i = 0; i < len; ++i) {
|
|
590
|
+
uint8_t byte = json[i];
|
|
591
|
+
tf_transform_code progress = json_work_progress(work, 1, 0, 0);
|
|
592
|
+
if (progress != TF_TRANSFORM_OK) return progress;
|
|
593
|
+
if (byte == 0) return tf_transform_set_error(
|
|
594
|
+
error, malformed_code, "JSON contains an embedded null byte");
|
|
595
|
+
if (in_string) {
|
|
596
|
+
if (escaped) {
|
|
597
|
+
if (byte == 'u' && i + 4 < len
|
|
598
|
+
&& json[i + 1] == '0' && json[i + 2] == '0'
|
|
599
|
+
&& json[i + 3] == '0' && json[i + 4] == '0')
|
|
600
|
+
return tf_transform_set_error(
|
|
601
|
+
error, malformed_code,
|
|
602
|
+
"JSON strings may not contain U+0000");
|
|
603
|
+
++current_string;
|
|
604
|
+
escaped = 0;
|
|
605
|
+
} else if (byte == '\\') {
|
|
606
|
+
++current_string;
|
|
607
|
+
escaped = 1;
|
|
608
|
+
}
|
|
609
|
+
else if (byte == '"') {
|
|
610
|
+
in_string = 0;
|
|
611
|
+
if (string_count == SIZE_MAX) return tf_transform_set_error(
|
|
612
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
613
|
+
"JSON string count overflows");
|
|
614
|
+
++string_count;
|
|
615
|
+
if (current_string > max_string) max_string = current_string;
|
|
616
|
+
if (current_string > limits->max_string_bytes)
|
|
617
|
+
return tf_transform_set_error(
|
|
618
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
619
|
+
"JSON string exceeds resource limit");
|
|
620
|
+
if (strings > SIZE_MAX - current_string)
|
|
621
|
+
return tf_transform_set_error(
|
|
622
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
623
|
+
"JSON string byte counter overflow");
|
|
624
|
+
strings += current_string;
|
|
625
|
+
if (strings > limits->max_decoded_string_bytes)
|
|
626
|
+
return tf_transform_set_error(
|
|
627
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
628
|
+
"JSON strings exceed resource limit");
|
|
629
|
+
last_string_end = i;
|
|
630
|
+
have_last_string = 1;
|
|
631
|
+
} else {
|
|
632
|
+
++current_string;
|
|
633
|
+
}
|
|
634
|
+
continue;
|
|
635
|
+
}
|
|
636
|
+
if (in_number) {
|
|
637
|
+
if ((byte >= '0' && byte <= '9') || byte == '+' || byte == '-'
|
|
638
|
+
|| byte == '.' || byte == 'e' || byte == 'E') {
|
|
639
|
+
if (current_number == SIZE_MAX) return tf_transform_set_error(
|
|
640
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
641
|
+
"JSON number length overflows");
|
|
642
|
+
++current_number;
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
in_number = 0;
|
|
646
|
+
if (number_count == SIZE_MAX) return tf_transform_set_error(
|
|
647
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
648
|
+
"JSON number count overflows");
|
|
649
|
+
++number_count;
|
|
650
|
+
if (current_number > max_number) max_number = current_number;
|
|
651
|
+
current_number = 0;
|
|
652
|
+
}
|
|
653
|
+
if (byte == '"') {
|
|
654
|
+
tf_transform_code code = scan_json_array_value(
|
|
655
|
+
depth, containers, array_expects_value,
|
|
656
|
+
category_arrays, category_counts, &array_elements,
|
|
657
|
+
&total_categories, limits, error);
|
|
658
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
659
|
+
expect_categories_array = 0;
|
|
660
|
+
have_last_string = 0;
|
|
661
|
+
in_string = 1;
|
|
662
|
+
current_string = 0;
|
|
663
|
+
last_string_start = i + 1;
|
|
664
|
+
} else if (byte == '{' || byte == '[') {
|
|
665
|
+
int is_category_array = byte == '[' && expect_categories_array;
|
|
666
|
+
tf_transform_code code = scan_json_array_value(
|
|
667
|
+
depth, containers, array_expects_value,
|
|
668
|
+
category_arrays, category_counts, &array_elements,
|
|
669
|
+
&total_categories, limits, error);
|
|
670
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
671
|
+
expect_categories_array = 0;
|
|
672
|
+
have_last_string = 0;
|
|
673
|
+
if (byte == '{') ++objects;
|
|
674
|
+
if (depth >= CJSON_NESTING_LIMIT
|
|
675
|
+
|| ++depth > limits->max_json_depth)
|
|
676
|
+
return tf_transform_set_error(
|
|
677
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON depth exceeds limit");
|
|
678
|
+
containers[depth - 1] = (char)byte;
|
|
679
|
+
array_expects_value[depth - 1] = byte == '[' ? 1u : 0u;
|
|
680
|
+
category_arrays[depth - 1] = is_category_array ? 1u : 0u;
|
|
681
|
+
category_counts[depth - 1] = 0;
|
|
682
|
+
} else if (byte == '}' || byte == ']') {
|
|
683
|
+
expect_categories_array = 0;
|
|
684
|
+
have_last_string = 0;
|
|
685
|
+
if (depth == 0) return tf_transform_set_error(
|
|
686
|
+
error, malformed_code, "unbalanced JSON structure");
|
|
687
|
+
if ((byte == '}' && containers[depth - 1] != '{')
|
|
688
|
+
|| (byte == ']' && containers[depth - 1] != '['))
|
|
689
|
+
return tf_transform_set_error(
|
|
690
|
+
error, malformed_code, "mismatched JSON structure");
|
|
691
|
+
--depth;
|
|
692
|
+
} else if (byte == ':') {
|
|
693
|
+
if (++keys > limits->max_object_keys)
|
|
694
|
+
return tf_transform_set_error(
|
|
695
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON key count exceeds limit");
|
|
696
|
+
expect_categories_array = have_last_string
|
|
697
|
+
&& json_string_token_equals_ascii(
|
|
698
|
+
json + last_string_start,
|
|
699
|
+
last_string_end - last_string_start, "categories");
|
|
700
|
+
have_last_string = 0;
|
|
701
|
+
} else if (byte == ',') {
|
|
702
|
+
expect_categories_array = 0;
|
|
703
|
+
have_last_string = 0;
|
|
704
|
+
if (depth != 0 && containers[depth - 1] == '[')
|
|
705
|
+
array_expects_value[depth - 1] = 1;
|
|
706
|
+
} else if (byte == '-' || (byte >= '0' && byte <= '9')) {
|
|
707
|
+
tf_transform_code code = scan_json_array_value(
|
|
708
|
+
depth, containers, array_expects_value,
|
|
709
|
+
category_arrays, category_counts, &array_elements,
|
|
710
|
+
&total_categories, limits, error);
|
|
711
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
712
|
+
expect_categories_array = 0;
|
|
713
|
+
have_last_string = 0;
|
|
714
|
+
in_number = 1;
|
|
715
|
+
current_number = 1;
|
|
716
|
+
} else if (byte == 't' || byte == 'f' || byte == 'n') {
|
|
717
|
+
tf_transform_code code = scan_json_array_value(
|
|
718
|
+
depth, containers, array_expects_value,
|
|
719
|
+
category_arrays, category_counts, &array_elements,
|
|
720
|
+
&total_categories, limits, error);
|
|
721
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
722
|
+
expect_categories_array = 0;
|
|
723
|
+
have_last_string = 0;
|
|
724
|
+
} else if (byte != ' ' && byte != '\t'
|
|
725
|
+
&& byte != '\r' && byte != '\n') {
|
|
726
|
+
expect_categories_array = 0;
|
|
727
|
+
have_last_string = 0;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
if (in_number) {
|
|
731
|
+
++number_count;
|
|
732
|
+
if (current_number > max_number) max_number = current_number;
|
|
733
|
+
}
|
|
734
|
+
if (in_string || depth != 0) return tf_transform_set_error(
|
|
735
|
+
error, malformed_code, "unterminated JSON structure");
|
|
736
|
+
if (max_number > 64) return tf_transform_set_error(
|
|
737
|
+
error, malformed_code,
|
|
738
|
+
"JSON numeric token exceeds bounded safe-integer syntax");
|
|
739
|
+
{
|
|
740
|
+
uint64_t nodes = 1;
|
|
741
|
+
uint64_t allocation_count;
|
|
742
|
+
uint64_t resident;
|
|
743
|
+
uint64_t persistent;
|
|
744
|
+
if ((uint64_t)keys > UINT64_MAX - nodes) goto metric_overflow;
|
|
745
|
+
nodes += (uint64_t)keys;
|
|
746
|
+
if ((uint64_t)array_elements > UINT64_MAX - nodes) goto metric_overflow;
|
|
747
|
+
nodes += (uint64_t)array_elements;
|
|
748
|
+
allocation_count = nodes;
|
|
749
|
+
if ((uint64_t)string_count > UINT64_MAX - allocation_count)
|
|
750
|
+
goto metric_overflow;
|
|
751
|
+
allocation_count += (uint64_t)string_count;
|
|
752
|
+
if ((uint64_t)number_count > UINT64_MAX - allocation_count)
|
|
753
|
+
goto metric_overflow;
|
|
754
|
+
allocation_count += (uint64_t)number_count;
|
|
755
|
+
if (nodes > UINT64_MAX / sizeof(cJSON)) goto metric_overflow;
|
|
756
|
+
persistent = nodes * sizeof(cJSON);
|
|
757
|
+
if ((uint64_t)strings > UINT64_MAX - persistent) goto metric_overflow;
|
|
758
|
+
persistent += (uint64_t)strings;
|
|
759
|
+
if ((uint64_t)string_count > UINT64_MAX - persistent)
|
|
760
|
+
goto metric_overflow;
|
|
761
|
+
persistent += (uint64_t)string_count;
|
|
762
|
+
resident = persistent;
|
|
763
|
+
if (number_count != 0
|
|
764
|
+
&& (uint64_t)max_number + 1 > UINT64_MAX - resident)
|
|
765
|
+
goto metric_overflow;
|
|
766
|
+
if (number_count != 0) resident += (uint64_t)max_number + 1;
|
|
767
|
+
if (allocation_count > limits->max_allocations_per_session
|
|
768
|
+
|| resident > limits->max_resident_state_bytes
|
|
769
|
+
|| sizeof(cJSON) > limits->max_allocation_bytes
|
|
770
|
+
|| (string_count != 0
|
|
771
|
+
&& (uint64_t)max_string + 1 > limits->max_allocation_bytes)
|
|
772
|
+
|| (number_count != 0
|
|
773
|
+
&& (uint64_t)max_number + 1 > limits->max_allocation_bytes))
|
|
774
|
+
return tf_transform_set_error(
|
|
775
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
776
|
+
"JSON parser growth exceeds session limits");
|
|
777
|
+
metrics->node_count_upper = nodes;
|
|
778
|
+
metrics->string_count = string_count;
|
|
779
|
+
metrics->number_count = number_count;
|
|
780
|
+
metrics->object_count = objects;
|
|
781
|
+
metrics->decoded_string_bytes = strings;
|
|
782
|
+
metrics->max_string_bytes = max_string;
|
|
783
|
+
metrics->max_number_bytes = max_number;
|
|
784
|
+
metrics->resident_bytes_persistent = persistent;
|
|
785
|
+
metrics->resident_bytes_upper = resident;
|
|
786
|
+
metrics->allocation_count_upper = allocation_count;
|
|
787
|
+
}
|
|
788
|
+
return TF_TRANSFORM_OK;
|
|
789
|
+
metric_overflow:
|
|
790
|
+
return tf_transform_set_error(
|
|
791
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
792
|
+
"JSON parser resource estimate overflows");
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
static int json_valid_utf8_with_work(
|
|
796
|
+
const uint8_t *data, size_t len, tf_json_work *work) {
|
|
797
|
+
size_t i = 0;
|
|
798
|
+
while (i < len) {
|
|
799
|
+
uint8_t c = data[i++];
|
|
800
|
+
uint32_t code;
|
|
801
|
+
size_t needed;
|
|
802
|
+
if (json_work_progress(work, 1, 0, 0) != TF_TRANSFORM_OK) return -1;
|
|
803
|
+
if (c <= 0x7f) {
|
|
804
|
+
if (c == 0) return 0;
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
if (c >= 0xc2 && c <= 0xdf) {
|
|
808
|
+
code = (uint32_t)(c & 0x1f);
|
|
809
|
+
needed = 1;
|
|
810
|
+
} else if (c >= 0xe0 && c <= 0xef) {
|
|
811
|
+
code = (uint32_t)(c & 0x0f);
|
|
812
|
+
needed = 2;
|
|
813
|
+
} else if (c >= 0xf0 && c <= 0xf4) {
|
|
814
|
+
code = (uint32_t)(c & 0x07);
|
|
815
|
+
needed = 3;
|
|
816
|
+
} else return 0;
|
|
817
|
+
if (needed > len - i) return 0;
|
|
818
|
+
for (size_t j = 0; j < needed; ++j) {
|
|
819
|
+
uint8_t next = data[i++];
|
|
820
|
+
if (json_work_progress(work, 1, 0, 0) != TF_TRANSFORM_OK) return -1;
|
|
821
|
+
if ((next & 0xc0) != 0x80) return 0;
|
|
822
|
+
code = (code << 6) | (uint32_t)(next & 0x3f);
|
|
823
|
+
}
|
|
824
|
+
if ((needed == 1 && code < 0x80)
|
|
825
|
+
|| (needed == 2 && code < 0x800)
|
|
826
|
+
|| (needed == 3 && code < 0x10000)
|
|
827
|
+
|| code > 0x10ffff
|
|
828
|
+
|| (code >= 0xd800 && code <= 0xdfff)) return 0;
|
|
829
|
+
}
|
|
830
|
+
return 1;
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
static tf_transform_code reject_duplicate_keys(
|
|
834
|
+
const cJSON *value, size_t depth, const tf_transform_limits_v1 *limits,
|
|
835
|
+
tf_json_work *work, tf_transform_error **error,
|
|
836
|
+
tf_transform_code malformed_code) {
|
|
837
|
+
const cJSON *child;
|
|
838
|
+
tf_transform_code code;
|
|
839
|
+
if (depth > limits->max_json_depth) return tf_transform_set_error(
|
|
840
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON depth exceeds limit");
|
|
841
|
+
if (cJSON_IsObject(value)) {
|
|
842
|
+
const cJSON **items = NULL;
|
|
843
|
+
size_t count = 0;
|
|
844
|
+
size_t index = 0;
|
|
845
|
+
cJSON_ArrayForEach(child, value) ++count;
|
|
846
|
+
if (count > 1) {
|
|
847
|
+
uint64_t bytes = (uint64_t)count * sizeof(items[0]);
|
|
848
|
+
code = json_work_allocation(work, bytes, error);
|
|
849
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
850
|
+
items = (const cJSON **)malloc((size_t)bytes);
|
|
851
|
+
if (!items) {
|
|
852
|
+
json_work_release(work, bytes);
|
|
853
|
+
return tf_transform_set_error(
|
|
854
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
855
|
+
"duplicate-key index allocation failed");
|
|
856
|
+
}
|
|
857
|
+
cJSON_ArrayForEach(child, value) items[index++] = child;
|
|
858
|
+
code = sort_object_items(items, count, work, error);
|
|
859
|
+
if (code == TF_TRANSFORM_OK) {
|
|
860
|
+
for (size_t i = 1; i < count; ++i) {
|
|
861
|
+
int compared;
|
|
862
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
863
|
+
if (code != TF_TRANSFORM_OK) break;
|
|
864
|
+
code = compare_object_keys(
|
|
865
|
+
items[i - 1], items[i], limits,
|
|
866
|
+
work, &compared, error);
|
|
867
|
+
if (code != TF_TRANSFORM_OK) break;
|
|
868
|
+
if (compared == 0) {
|
|
869
|
+
code = tf_transform_set_error(
|
|
870
|
+
error, malformed_code,
|
|
871
|
+
"duplicate JSON object key");
|
|
872
|
+
break;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
free(items);
|
|
877
|
+
json_work_release(work, bytes);
|
|
878
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
cJSON_ArrayForEach(child, value) {
|
|
882
|
+
code = json_work_progress(work, 0, 1, 0);
|
|
883
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
884
|
+
code = reject_duplicate_keys(
|
|
885
|
+
child, depth + 1, limits, work, error, malformed_code);
|
|
886
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
887
|
+
}
|
|
888
|
+
return TF_TRANSFORM_OK;
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
static void delete_json_runtime_walk(
|
|
892
|
+
cJSON *item, const tf_transform_runtime_copy *runtime,
|
|
893
|
+
size_t *iterations, int *cancelled) {
|
|
894
|
+
while (item) {
|
|
895
|
+
cJSON *next = item->next;
|
|
896
|
+
if (!(item->type & cJSON_IsReference) && item->child)
|
|
897
|
+
delete_json_runtime_walk(
|
|
898
|
+
item->child, runtime, iterations, cancelled);
|
|
899
|
+
if (!(item->type & cJSON_IsReference) && item->valuestring)
|
|
900
|
+
cJSON_free(item->valuestring);
|
|
901
|
+
if (!(item->type & cJSON_StringIsConst) && item->string)
|
|
902
|
+
cJSON_free(item->string);
|
|
903
|
+
cJSON_free(item);
|
|
904
|
+
++*iterations;
|
|
905
|
+
if (*iterations % TF_TRANSFORM_CANCEL_ITERS_V1 == 0
|
|
906
|
+
&& runtime && runtime->cancel
|
|
907
|
+
&& runtime->cancel(runtime->cancel_user))
|
|
908
|
+
*cancelled = 1;
|
|
909
|
+
item = next;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
static int delete_json_runtime(
|
|
914
|
+
cJSON *item, const tf_transform_runtime_copy *runtime) {
|
|
915
|
+
size_t iterations = 0;
|
|
916
|
+
int cancelled = 0;
|
|
917
|
+
if (runtime && runtime->cancel && runtime->cancel(runtime->cancel_user))
|
|
918
|
+
cancelled = 1;
|
|
919
|
+
delete_json_runtime_walk(item, runtime, &iterations, &cancelled);
|
|
920
|
+
if (runtime && runtime->cancel && runtime->cancel(runtime->cancel_user))
|
|
921
|
+
cancelled = 1;
|
|
922
|
+
return cancelled;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
static tf_transform_code json_parse_bounded_common(
|
|
926
|
+
const uint8_t *json, size_t json_len,
|
|
927
|
+
const tf_transform_runtime_copy *runtime,
|
|
928
|
+
tf_transform_resource_ledger *shared_ledger, cJSON **out,
|
|
929
|
+
uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
|
|
930
|
+
tf_transform_error **error, tf_transform_code malformed_code) {
|
|
931
|
+
cJSON *value;
|
|
932
|
+
const char *parse_end = NULL;
|
|
933
|
+
tf_transform_code code;
|
|
934
|
+
tf_json_work work = {0};
|
|
935
|
+
tf_transform_resource_ledger local_ledger;
|
|
936
|
+
tf_transform_resource_ledger *ledger;
|
|
937
|
+
tf_json_scan_metrics metrics;
|
|
938
|
+
uint64_t allocation_start;
|
|
939
|
+
int valid_utf8;
|
|
940
|
+
if (out) *out = NULL;
|
|
941
|
+
if (resident_bytes_out) *resident_bytes_out = 0;
|
|
942
|
+
if (allocation_count_out) *allocation_count_out = 0;
|
|
943
|
+
if (!json || json_len == 0 || !runtime || !out)
|
|
944
|
+
return tf_transform_set_error(
|
|
945
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "JSON argument is null or empty");
|
|
946
|
+
work.runtime = runtime;
|
|
947
|
+
if (shared_ledger) ledger = shared_ledger;
|
|
948
|
+
else {
|
|
949
|
+
tf_transform_resource_ledger_init(&local_ledger, runtime);
|
|
950
|
+
ledger = &local_ledger;
|
|
951
|
+
}
|
|
952
|
+
work.ledger = ledger;
|
|
953
|
+
work.error = error;
|
|
954
|
+
allocation_start = ledger->allocation_count;
|
|
955
|
+
code = json_work_progress(&work, 0, 0, 1);
|
|
956
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
957
|
+
if (json_len > runtime->limits.max_plan_bytes || json_len > SIZE_MAX - 1)
|
|
958
|
+
return tf_transform_set_error(
|
|
959
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "JSON input exceeds byte limit");
|
|
960
|
+
valid_utf8 = json_valid_utf8_with_work(json, json_len, &work);
|
|
961
|
+
if (valid_utf8 < 0) return TF_TRANSFORM_CANCELLED;
|
|
962
|
+
if (!valid_utf8)
|
|
963
|
+
return tf_transform_set_error(
|
|
964
|
+
error, malformed_code, "JSON is not valid UTF-8");
|
|
965
|
+
code = scan_json_bounds(
|
|
966
|
+
json, json_len, &runtime->limits, &work, &metrics,
|
|
967
|
+
error, malformed_code);
|
|
968
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
969
|
+
code = tf_transform_resource_charge_batch(
|
|
970
|
+
ledger, metrics.allocation_count_upper,
|
|
971
|
+
metrics.resident_bytes_persistent, metrics.resident_bytes_upper,
|
|
972
|
+
error);
|
|
973
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
974
|
+
value = cJSON_ParseWithLengthOptsAndPoll(
|
|
975
|
+
(const char *)json, json_len, &parse_end, 0,
|
|
976
|
+
json_parser_poll, &work, TF_TRANSFORM_CANCEL_BYTES_V1,
|
|
977
|
+
TF_TRANSFORM_CANCEL_ITERS_V1);
|
|
978
|
+
if (!value && work.cancelled) return tf_transform_set_error(
|
|
979
|
+
error, TF_TRANSFORM_CANCELLED, "prepared transform cancelled");
|
|
980
|
+
if (!value) return tf_transform_set_error(
|
|
981
|
+
error, cJSON_ParseHadAllocationFailure()
|
|
982
|
+
? TF_TRANSFORM_ALLOCATION : malformed_code,
|
|
983
|
+
cJSON_ParseHadAllocationFailure()
|
|
984
|
+
? "JSON parser allocation failed" : "invalid JSON");
|
|
985
|
+
while ((size_t)(parse_end - (const char *)json) < json_len
|
|
986
|
+
&& (*parse_end == ' ' || *parse_end == '\t'
|
|
987
|
+
|| *parse_end == '\r' || *parse_end == '\n')) {
|
|
988
|
+
code = json_work_progress(&work, 1, 0, 0);
|
|
989
|
+
if (code != TF_TRANSFORM_OK) {
|
|
990
|
+
(void)delete_json_runtime(value, runtime);
|
|
991
|
+
return code;
|
|
992
|
+
}
|
|
993
|
+
++parse_end;
|
|
994
|
+
}
|
|
995
|
+
if ((size_t)(parse_end - (const char *)json) != json_len) {
|
|
996
|
+
(void)delete_json_runtime(value, runtime);
|
|
997
|
+
return tf_transform_set_error(
|
|
998
|
+
error, malformed_code, "JSON contains trailing data");
|
|
999
|
+
}
|
|
1000
|
+
code = reject_duplicate_keys(
|
|
1001
|
+
value, 1, &runtime->limits, &work, error, malformed_code);
|
|
1002
|
+
if (code != TF_TRANSFORM_OK) {
|
|
1003
|
+
(void)delete_json_runtime(value, runtime);
|
|
1004
|
+
return code;
|
|
1005
|
+
}
|
|
1006
|
+
code = json_work_progress(&work, 0, 0, 1);
|
|
1007
|
+
if (code != TF_TRANSFORM_OK) {
|
|
1008
|
+
(void)delete_json_runtime(value, runtime);
|
|
1009
|
+
return code;
|
|
1010
|
+
}
|
|
1011
|
+
*out = value;
|
|
1012
|
+
if (resident_bytes_out)
|
|
1013
|
+
*resident_bytes_out = metrics.resident_bytes_persistent;
|
|
1014
|
+
if (allocation_count_out)
|
|
1015
|
+
*allocation_count_out = ledger->allocation_count - allocation_start;
|
|
1016
|
+
return TF_TRANSFORM_OK;
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
tf_transform_code tf_transform_json_parse_bounded_runtime(
|
|
1020
|
+
const uint8_t *json, size_t json_len,
|
|
1021
|
+
const tf_transform_runtime_copy *runtime, cJSON **out,
|
|
1022
|
+
uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
|
|
1023
|
+
tf_transform_error **error, tf_transform_code malformed_code) {
|
|
1024
|
+
return json_parse_bounded_common(
|
|
1025
|
+
json, json_len, runtime, NULL, out,
|
|
1026
|
+
resident_bytes_out, allocation_count_out,
|
|
1027
|
+
error, malformed_code);
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
tf_transform_code tf_transform_json_parse_bounded_runtime_ledger(
|
|
1031
|
+
const uint8_t *json, size_t json_len,
|
|
1032
|
+
const tf_transform_runtime_copy *runtime,
|
|
1033
|
+
tf_transform_resource_ledger *ledger, cJSON **out,
|
|
1034
|
+
uint64_t *resident_bytes_out, uint64_t *allocation_count_out,
|
|
1035
|
+
tf_transform_error **error, tf_transform_code malformed_code) {
|
|
1036
|
+
if (!ledger || ledger->runtime != runtime) {
|
|
1037
|
+
if (out) *out = NULL;
|
|
1038
|
+
return tf_transform_set_error(
|
|
1039
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
1040
|
+
"JSON parser resource ledger is invalid");
|
|
1041
|
+
}
|
|
1042
|
+
return json_parse_bounded_common(
|
|
1043
|
+
json, json_len, runtime, ledger, out,
|
|
1044
|
+
resident_bytes_out, allocation_count_out,
|
|
1045
|
+
error, malformed_code);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
tf_transform_code tf_transform_json_parse_bounded(
|
|
1049
|
+
const uint8_t *json, size_t json_len,
|
|
1050
|
+
const tf_transform_limits_v1 *limits, cJSON **out,
|
|
1051
|
+
tf_transform_error **error, tf_transform_code malformed_code) {
|
|
1052
|
+
tf_transform_runtime_copy runtime;
|
|
1053
|
+
if (!limits) {
|
|
1054
|
+
if (out) *out = NULL;
|
|
1055
|
+
return tf_transform_set_error(
|
|
1056
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "JSON limits are null");
|
|
1057
|
+
}
|
|
1058
|
+
memset(&runtime, 0, sizeof(runtime));
|
|
1059
|
+
runtime.limits = *limits;
|
|
1060
|
+
return json_parse_bounded_common(
|
|
1061
|
+
json, json_len, &runtime, NULL, out, NULL, NULL,
|
|
1062
|
+
error, malformed_code);
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
static int exact_keys(const cJSON *object, const char *const *keys, size_t count) {
|
|
1066
|
+
const cJSON *child;
|
|
1067
|
+
size_t seen = 0;
|
|
1068
|
+
if (!cJSON_IsObject(object)) return 0;
|
|
1069
|
+
cJSON_ArrayForEach(child, object) {
|
|
1070
|
+
int found = 0;
|
|
1071
|
+
for (size_t i = 0; i < count; ++i) {
|
|
1072
|
+
if (strcmp(child->string, keys[i]) == 0) { found = 1; break; }
|
|
1073
|
+
}
|
|
1074
|
+
if (!found) return 0;
|
|
1075
|
+
++seen;
|
|
1076
|
+
}
|
|
1077
|
+
return seen == count;
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
static const cJSON *required_item(const cJSON *object, const char *name) {
|
|
1081
|
+
return cJSON_GetObjectItemCaseSensitive(object, name);
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
static int json_string_equals(const cJSON *value, const char *expected) {
|
|
1085
|
+
return cJSON_IsString(value) && value->valuestring
|
|
1086
|
+
&& strcmp(value->valuestring, expected) == 0;
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1089
|
+
static int safe_uint64(const cJSON *value, uint64_t *out) {
|
|
1090
|
+
double integer;
|
|
1091
|
+
if (!cJSON_IsNumber(value) || !isfinite(value->valuedouble)
|
|
1092
|
+
|| value->valuedouble < 0
|
|
1093
|
+
|| modf(value->valuedouble, &integer) != 0.0
|
|
1094
|
+
|| integer >= 9007199254740992.0) return 0;
|
|
1095
|
+
*out = (uint64_t)integer;
|
|
1096
|
+
return 1;
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
static int safe_int64(const cJSON *value, int64_t *out) {
|
|
1100
|
+
double integer;
|
|
1101
|
+
if (!cJSON_IsNumber(value) || !isfinite(value->valuedouble)
|
|
1102
|
+
|| modf(value->valuedouble, &integer) != 0.0
|
|
1103
|
+
|| integer <= -9007199254740992.0
|
|
1104
|
+
|| integer >= 9007199254740992.0) return 0;
|
|
1105
|
+
*out = (int64_t)integer;
|
|
1106
|
+
return 1;
|
|
1107
|
+
}
|
|
1108
|
+
|
|
1109
|
+
static tf_transform_code runtime_cstring_length(
|
|
1110
|
+
const char *text, size_t maximum,
|
|
1111
|
+
const tf_transform_runtime_copy *runtime,
|
|
1112
|
+
size_t *out, tf_transform_error **error) {
|
|
1113
|
+
size_t length = 0;
|
|
1114
|
+
if (!text || !runtime || !out)
|
|
1115
|
+
return tf_transform_set_error(
|
|
1116
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
1117
|
+
"runtime string argument is null");
|
|
1118
|
+
while (length <= maximum && text[length] != '\0') {
|
|
1119
|
+
if (length % TF_TRANSFORM_CANCEL_BYTES_V1 == 0) {
|
|
1120
|
+
tf_transform_code code = tf_transform_poll_cancel(runtime, error);
|
|
1121
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1122
|
+
}
|
|
1123
|
+
++length;
|
|
1124
|
+
}
|
|
1125
|
+
if (length > maximum)
|
|
1126
|
+
return tf_transform_set_error(
|
|
1127
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1128
|
+
"runtime string exceeds byte limit");
|
|
1129
|
+
*out = length;
|
|
1130
|
+
return tf_transform_poll_cancel(runtime, error);
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
static tf_transform_code runtime_span_equal(
|
|
1134
|
+
const char *left, size_t left_len,
|
|
1135
|
+
const char *right, size_t right_len,
|
|
1136
|
+
const tf_transform_runtime_copy *runtime,
|
|
1137
|
+
int *equal, tf_transform_error **error) {
|
|
1138
|
+
int comparison = 0;
|
|
1139
|
+
tf_transform_code code;
|
|
1140
|
+
if (!left || !right || !runtime || !equal)
|
|
1141
|
+
return tf_transform_set_error(
|
|
1142
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
1143
|
+
"runtime span argument is null");
|
|
1144
|
+
code = tf_transform_compare_bytes_runtime(
|
|
1145
|
+
left, left_len, right, right_len,
|
|
1146
|
+
runtime, &comparison, error);
|
|
1147
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1148
|
+
*equal = comparison == 0;
|
|
1149
|
+
return TF_TRANSFORM_OK;
|
|
1150
|
+
}
|
|
1151
|
+
|
|
1152
|
+
static int parse_tagged_bits(
|
|
1153
|
+
const cJSON *value, uint64_t *out, uint32_t *dtype) {
|
|
1154
|
+
static const char *const keys[] = {"t", "v"};
|
|
1155
|
+
const cJSON *tag;
|
|
1156
|
+
const cJSON *bits;
|
|
1157
|
+
uint64_t decoded = 0;
|
|
1158
|
+
size_t digits;
|
|
1159
|
+
if (!out || !dtype || !exact_keys(value, keys, 2)) return 0;
|
|
1160
|
+
tag = required_item(value, "t");
|
|
1161
|
+
bits = required_item(value, "v");
|
|
1162
|
+
if (!cJSON_IsString(tag) || !tag->valuestring
|
|
1163
|
+
|| !cJSON_IsString(bits) || !bits->valuestring) return 0;
|
|
1164
|
+
if (strcmp(tag->valuestring, "f64") == 0) {
|
|
1165
|
+
*dtype = TF_VIEW_FLOAT64;
|
|
1166
|
+
digits = 16;
|
|
1167
|
+
} else if (strcmp(tag->valuestring, "f32") == 0) {
|
|
1168
|
+
*dtype = TF_VIEW_FLOAT32;
|
|
1169
|
+
digits = 8;
|
|
1170
|
+
} else return 0;
|
|
1171
|
+
for (size_t i = 0; i < digits; ++i) {
|
|
1172
|
+
unsigned char c = (unsigned char)bits->valuestring[i];
|
|
1173
|
+
unsigned int nibble;
|
|
1174
|
+
if (c == '\0') return 0;
|
|
1175
|
+
if (c >= '0' && c <= '9') nibble = c - '0';
|
|
1176
|
+
else if (c >= 'a' && c <= 'f') nibble = c - 'a' + 10u;
|
|
1177
|
+
else return 0;
|
|
1178
|
+
decoded = (decoded << 4) | nibble;
|
|
1179
|
+
}
|
|
1180
|
+
if (bits->valuestring[digits] != '\0') return 0;
|
|
1181
|
+
*out = decoded;
|
|
1182
|
+
return 1;
|
|
1183
|
+
}
|
|
1184
|
+
|
|
1185
|
+
static int parse_tagged_constant(const cJSON *value, double *out, uint32_t *dtype) {
|
|
1186
|
+
uint64_t decoded = 0;
|
|
1187
|
+
if (!out || !parse_tagged_bits(value, &decoded, dtype)) return 0;
|
|
1188
|
+
if (*dtype == TF_VIEW_FLOAT64) *out = tf_transform_double_from_bits(decoded);
|
|
1189
|
+
else {
|
|
1190
|
+
uint32_t fbits = (uint32_t)decoded;
|
|
1191
|
+
float fvalue;
|
|
1192
|
+
memcpy(&fvalue, &fbits, sizeof(fvalue));
|
|
1193
|
+
*out = (double)fvalue;
|
|
1194
|
+
}
|
|
1195
|
+
return tf_transform_double_is_finite(*out);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
static int parse_numeric_recipe_branch(
|
|
1199
|
+
const cJSON *numeric, tf_transform_recipe_column *out) {
|
|
1200
|
+
static const char *const numeric_keys[] = {"impute", "normalize"};
|
|
1201
|
+
static const char *const impute_keys[] = {"allMissing", "constant", "op"};
|
|
1202
|
+
static const char *const normalize_keys[] = {"ddof", "op"};
|
|
1203
|
+
const cJSON *impute;
|
|
1204
|
+
const cJSON *normalize;
|
|
1205
|
+
const cJSON *op;
|
|
1206
|
+
const cJSON *policy;
|
|
1207
|
+
const cJSON *constant;
|
|
1208
|
+
if (!exact_keys(numeric, numeric_keys, 2)) return 0;
|
|
1209
|
+
impute = required_item(numeric, "impute");
|
|
1210
|
+
if (!exact_keys(impute, impute_keys, 3)) return 0;
|
|
1211
|
+
op = required_item(impute, "op");
|
|
1212
|
+
policy = required_item(impute, "allMissing");
|
|
1213
|
+
constant = required_item(impute, "constant");
|
|
1214
|
+
if (json_string_equals(op, "none")) {
|
|
1215
|
+
if (!cJSON_IsNull(policy) || !cJSON_IsNull(constant)) return 0;
|
|
1216
|
+
out->impute = TF_TRANSFORM_IMPUTE_NONE;
|
|
1217
|
+
} else if (json_string_equals(op, "zero")) {
|
|
1218
|
+
if (!cJSON_IsNull(policy) || !cJSON_IsNull(constant)) return 0;
|
|
1219
|
+
out->impute = TF_TRANSFORM_IMPUTE_ZERO;
|
|
1220
|
+
out->constant = 0.0;
|
|
1221
|
+
} else if (json_string_equals(op, "constant")) {
|
|
1222
|
+
if (!cJSON_IsNull(policy)
|
|
1223
|
+
|| !parse_tagged_constant(
|
|
1224
|
+
constant, &out->constant, &out->constant_dtype)) return 0;
|
|
1225
|
+
out->impute = TF_TRANSFORM_IMPUTE_CONSTANT;
|
|
1226
|
+
} else if (json_string_equals(op, "mean")
|
|
1227
|
+
|| json_string_equals(op, "median")) {
|
|
1228
|
+
if (!cJSON_IsNull(constant)) return 0;
|
|
1229
|
+
if (json_string_equals(policy, "error"))
|
|
1230
|
+
out->all_missing = TF_TRANSFORM_ALL_MISSING_ERROR;
|
|
1231
|
+
else if (json_string_equals(policy, "zero"))
|
|
1232
|
+
out->all_missing = TF_TRANSFORM_ALL_MISSING_ZERO;
|
|
1233
|
+
else return 0;
|
|
1234
|
+
out->impute = json_string_equals(op, "mean")
|
|
1235
|
+
? TF_TRANSFORM_IMPUTE_MEAN : TF_TRANSFORM_IMPUTE_MEDIAN;
|
|
1236
|
+
} else return 0;
|
|
1237
|
+
normalize = required_item(numeric, "normalize");
|
|
1238
|
+
if (!exact_keys(normalize, normalize_keys, 2)) return 0;
|
|
1239
|
+
op = required_item(normalize, "op");
|
|
1240
|
+
if (json_string_equals(op, "none")) {
|
|
1241
|
+
if (!cJSON_IsNull(required_item(normalize, "ddof"))) return 0;
|
|
1242
|
+
out->normalize = TF_TRANSFORM_NORMALIZE_NONE;
|
|
1243
|
+
} else if (json_string_equals(op, "minmax")) {
|
|
1244
|
+
if (!cJSON_IsNull(required_item(normalize, "ddof"))) return 0;
|
|
1245
|
+
out->normalize = TF_TRANSFORM_NORMALIZE_MINMAX;
|
|
1246
|
+
} else if (json_string_equals(op, "standard")) {
|
|
1247
|
+
uint64_t ddof;
|
|
1248
|
+
if (!safe_uint64(required_item(normalize, "ddof"), &ddof) || ddof > 1)
|
|
1249
|
+
return 0;
|
|
1250
|
+
out->normalize = TF_TRANSFORM_NORMALIZE_STANDARD;
|
|
1251
|
+
out->ddof = (int)ddof;
|
|
1252
|
+
} else return 0;
|
|
1253
|
+
return 1;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
static int parse_categorical_recipe_branch(
|
|
1257
|
+
const cJSON *categorical, tf_transform_recipe_column *out) {
|
|
1258
|
+
static const char *const categorical_keys[] = {"encode", "impute"};
|
|
1259
|
+
static const char *const impute_keys[] = {"allMissing", "constant", "op"};
|
|
1260
|
+
static const char *const encode_keys[] = {
|
|
1261
|
+
"categories", "op", "sentinelLabel", "unknown"
|
|
1262
|
+
};
|
|
1263
|
+
const cJSON *impute;
|
|
1264
|
+
const cJSON *encode;
|
|
1265
|
+
const cJSON *categories;
|
|
1266
|
+
const cJSON *impute_op;
|
|
1267
|
+
const cJSON *policy;
|
|
1268
|
+
if (!exact_keys(categorical, categorical_keys, 2)) return 0;
|
|
1269
|
+
impute = required_item(categorical, "impute");
|
|
1270
|
+
encode = required_item(categorical, "encode");
|
|
1271
|
+
if (!exact_keys(impute, impute_keys, 3)
|
|
1272
|
+
|| !cJSON_IsNull(required_item(impute, "constant"))) return 0;
|
|
1273
|
+
impute_op = required_item(impute, "op");
|
|
1274
|
+
policy = required_item(impute, "allMissing");
|
|
1275
|
+
if (json_string_equals(impute_op, "none")) {
|
|
1276
|
+
if (!cJSON_IsNull(policy)) return 0;
|
|
1277
|
+
out->categorical_impute = TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE;
|
|
1278
|
+
out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_NONE;
|
|
1279
|
+
} else if (json_string_equals(impute_op, "mode")) {
|
|
1280
|
+
if (json_string_equals(policy, "error"))
|
|
1281
|
+
out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_ERROR;
|
|
1282
|
+
else if (json_string_equals(policy, "zero"))
|
|
1283
|
+
out->categorical_all_missing = TF_TRANSFORM_ALL_MISSING_ZERO;
|
|
1284
|
+
else return 0;
|
|
1285
|
+
out->categorical_impute = TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE;
|
|
1286
|
+
} else return 0;
|
|
1287
|
+
if (!exact_keys(encode, encode_keys, 4)) return 0;
|
|
1288
|
+
categories = required_item(encode, "categories");
|
|
1289
|
+
if (json_string_equals(required_item(encode, "op"), "none")) {
|
|
1290
|
+
if (!cJSON_IsNull(required_item(encode, "sentinelLabel"))
|
|
1291
|
+
|| !cJSON_IsNull(required_item(encode, "unknown"))) return 0;
|
|
1292
|
+
out->categorical_encode = TF_TRANSFORM_ENCODE_NONE;
|
|
1293
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_NONE;
|
|
1294
|
+
} else if (json_string_equals(required_item(encode, "op"), "label")) {
|
|
1295
|
+
const cJSON *unknown = required_item(encode, "unknown");
|
|
1296
|
+
const cJSON *sentinel = required_item(encode, "sentinelLabel");
|
|
1297
|
+
out->categorical_encode = TF_TRANSFORM_ENCODE_LABEL;
|
|
1298
|
+
if (json_string_equals(unknown, "error") && cJSON_IsNull(sentinel))
|
|
1299
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ERROR;
|
|
1300
|
+
else if (json_string_equals(unknown, "other") && cJSON_IsNull(sentinel))
|
|
1301
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_OTHER;
|
|
1302
|
+
else if (json_string_equals(unknown, "sentinel")
|
|
1303
|
+
&& safe_int64(sentinel, &out->categorical_sentinel_label)) {
|
|
1304
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_SENTINEL;
|
|
1305
|
+
out->categorical_has_sentinel_label = 1;
|
|
1306
|
+
} else return 0;
|
|
1307
|
+
} else if (json_string_equals(required_item(encode, "op"), "onehot")) {
|
|
1308
|
+
const cJSON *unknown = required_item(encode, "unknown");
|
|
1309
|
+
if (!cJSON_IsNull(required_item(encode, "sentinelLabel"))) return 0;
|
|
1310
|
+
out->categorical_encode = TF_TRANSFORM_ENCODE_ONEHOT;
|
|
1311
|
+
if (json_string_equals(unknown, "error"))
|
|
1312
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ERROR;
|
|
1313
|
+
else if (json_string_equals(unknown, "all_zero"))
|
|
1314
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_ALL_ZERO;
|
|
1315
|
+
else if (json_string_equals(unknown, "other"))
|
|
1316
|
+
out->categorical_unknown = TF_TRANSFORM_UNKNOWN_OTHER;
|
|
1317
|
+
else return 0;
|
|
1318
|
+
} else return 0;
|
|
1319
|
+
if (out->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
1320
|
+
&& out->categorical_encode == TF_TRANSFORM_ENCODE_NONE) {
|
|
1321
|
+
if (!cJSON_IsNull(categories)) return 0;
|
|
1322
|
+
out->categorical_discover = 0;
|
|
1323
|
+
} else {
|
|
1324
|
+
if (!json_string_equals(categories, "discover") && !cJSON_IsArray(categories)) return 0;
|
|
1325
|
+
out->categorical_discover = !cJSON_IsArray(categories);
|
|
1326
|
+
}
|
|
1327
|
+
return 1;
|
|
1328
|
+
}
|
|
1329
|
+
|
|
1330
|
+
static int tagged_category_bits(
|
|
1331
|
+
const cJSON *value, uint32_t expected_dtype, uint64_t *out);
|
|
1332
|
+
|
|
1333
|
+
static tf_transform_code parse_fixed_categories(
|
|
1334
|
+
const cJSON *categories, const tf_transform_runtime_copy *runtime,
|
|
1335
|
+
tf_transform_resource_ledger *ledger, tf_transform_recipe_column *out,
|
|
1336
|
+
tf_transform_error **error) {
|
|
1337
|
+
size_t count = 0;
|
|
1338
|
+
uint32_t dtype;
|
|
1339
|
+
uint64_t previous = 0;
|
|
1340
|
+
const cJSON *entry;
|
|
1341
|
+
tf_transform_code code;
|
|
1342
|
+
if (out->kind != TF_TRANSFORM_KIND_CATEGORICAL || !categories->child)
|
|
1343
|
+
goto invalid;
|
|
1344
|
+
dtype = json_string_equals(required_item(categories->child, "t"), "f32")
|
|
1345
|
+
? TF_VIEW_FLOAT32 : TF_VIEW_FLOAT64;
|
|
1346
|
+
cJSON_ArrayForEach(entry, categories) {
|
|
1347
|
+
uint64_t bits;
|
|
1348
|
+
if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
1349
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
1350
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1351
|
+
}
|
|
1352
|
+
if (!tagged_category_bits(entry, dtype, &bits)
|
|
1353
|
+
|| (count && tf_transform_category_compare(previous, bits, dtype) >= 0))
|
|
1354
|
+
goto invalid;
|
|
1355
|
+
previous = bits;
|
|
1356
|
+
if ((uint64_t)count >= runtime->limits.max_categories_per_column
|
|
1357
|
+
|| (uint64_t)count >= runtime->limits.max_total_categories
|
|
1358
|
+
|| count >= SIZE_MAX / sizeof(*out->categorical_fixed))
|
|
1359
|
+
return tf_transform_set_error(error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1360
|
+
"fixed dictionary exceeds category limits");
|
|
1361
|
+
count++;
|
|
1362
|
+
}
|
|
1363
|
+
out->categorical_fixed = tf_transform_resource_calloc(
|
|
1364
|
+
ledger, count, sizeof(*out->categorical_fixed), error);
|
|
1365
|
+
if (!out->categorical_fixed)
|
|
1366
|
+
return ledger->last_code != TF_TRANSFORM_OK ? ledger->last_code
|
|
1367
|
+
: tf_transform_set_error(error, TF_TRANSFORM_ALLOCATION,
|
|
1368
|
+
"fixed dictionary allocation failed");
|
|
1369
|
+
out->categorical_fixed_count = count;
|
|
1370
|
+
out->categorical_fixed_dtype = dtype;
|
|
1371
|
+
size_t i = 0;
|
|
1372
|
+
cJSON_ArrayForEach(entry, categories) {
|
|
1373
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
1374
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
1375
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1376
|
+
}
|
|
1377
|
+
if (!tagged_category_bits(entry, dtype, &out->categorical_fixed[i++].bits))
|
|
1378
|
+
goto invalid;
|
|
1379
|
+
}
|
|
1380
|
+
return TF_TRANSFORM_OK;
|
|
1381
|
+
invalid:
|
|
1382
|
+
return tf_transform_set_error(error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1383
|
+
"fixed categories require a declared categorical column and sorted unique finite matching tags");
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
static tf_transform_code parse_recipe_column(
|
|
1387
|
+
const cJSON *value, const tf_transform_runtime_copy *runtime,
|
|
1388
|
+
tf_transform_resource_ledger *ledger,
|
|
1389
|
+
tf_transform_recipe_column *out, tf_transform_error **error) {
|
|
1390
|
+
static const char *const column_keys[] = {
|
|
1391
|
+
"categorical", "kind", "numeric", "sourceId"
|
|
1392
|
+
};
|
|
1393
|
+
static const char *const kind_keys[] = {
|
|
1394
|
+
"maxCategories", "op", "rule", "value"
|
|
1395
|
+
};
|
|
1396
|
+
const cJSON *kind;
|
|
1397
|
+
const cJSON *numeric;
|
|
1398
|
+
const cJSON *categorical;
|
|
1399
|
+
const cJSON *source_id;
|
|
1400
|
+
const cJSON *kind_op;
|
|
1401
|
+
const cJSON *kind_value;
|
|
1402
|
+
uint64_t infer_max_categories = 0;
|
|
1403
|
+
int has_numeric = 0;
|
|
1404
|
+
int has_categorical = 0;
|
|
1405
|
+
size_t source_len;
|
|
1406
|
+
memset(out, 0, sizeof(*out));
|
|
1407
|
+
const tf_transform_limits_v1 *limits = &runtime->limits;
|
|
1408
|
+
tf_transform_code code;
|
|
1409
|
+
if (!exact_keys(value, column_keys, 4))
|
|
1410
|
+
return tf_transform_set_error(
|
|
1411
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "recipe column shape is invalid");
|
|
1412
|
+
kind = required_item(value, "kind");
|
|
1413
|
+
if (!exact_keys(kind, kind_keys, 4))
|
|
1414
|
+
return tf_transform_set_error(
|
|
1415
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1416
|
+
"prepared transform kind shape is invalid");
|
|
1417
|
+
kind_op = required_item(kind, "op");
|
|
1418
|
+
kind_value = required_item(kind, "value");
|
|
1419
|
+
if (json_string_equals(kind_op, "declared")) {
|
|
1420
|
+
if (!cJSON_IsNull(required_item(kind, "maxCategories"))
|
|
1421
|
+
|| !cJSON_IsNull(required_item(kind, "rule")))
|
|
1422
|
+
return tf_transform_set_error(
|
|
1423
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1424
|
+
"declared transform kind is invalid");
|
|
1425
|
+
if (json_string_equals(kind_value, "numeric")) {
|
|
1426
|
+
out->kind = TF_TRANSFORM_KIND_NUMERIC;
|
|
1427
|
+
has_numeric = 1;
|
|
1428
|
+
} else if (json_string_equals(kind_value, "categorical")) {
|
|
1429
|
+
out->kind = TF_TRANSFORM_KIND_CATEGORICAL;
|
|
1430
|
+
has_categorical = 1;
|
|
1431
|
+
} else return tf_transform_set_error(
|
|
1432
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1433
|
+
"declared transform kind value is invalid");
|
|
1434
|
+
} else if (json_string_equals(kind_op, "infer")) {
|
|
1435
|
+
if (!cJSON_IsNull(kind_value)
|
|
1436
|
+
|| !json_string_equals(
|
|
1437
|
+
required_item(kind, "rule"),
|
|
1438
|
+
"finite-integer-cardinality-v1")
|
|
1439
|
+
|| !safe_uint64(
|
|
1440
|
+
required_item(kind, "maxCategories"), &infer_max_categories)
|
|
1441
|
+
|| infer_max_categories < 2)
|
|
1442
|
+
return tf_transform_set_error(
|
|
1443
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1444
|
+
"inferred transform kind is invalid");
|
|
1445
|
+
if (infer_max_categories > limits->max_categories_per_column
|
|
1446
|
+
|| infer_max_categories > limits->max_total_categories)
|
|
1447
|
+
return tf_transform_set_error(
|
|
1448
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1449
|
+
"inferred category bound exceeds host limits");
|
|
1450
|
+
out->kind = TF_TRANSFORM_KIND_INFER;
|
|
1451
|
+
out->infer_max_categories = infer_max_categories;
|
|
1452
|
+
has_numeric = 1;
|
|
1453
|
+
has_categorical = 1;
|
|
1454
|
+
} else return tf_transform_set_error(
|
|
1455
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1456
|
+
"transform kind operation is unsupported");
|
|
1457
|
+
source_id = required_item(value, "sourceId");
|
|
1458
|
+
if (!cJSON_IsString(source_id) || !source_id->valuestring)
|
|
1459
|
+
return tf_transform_set_error(
|
|
1460
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe sourceId");
|
|
1461
|
+
code = runtime_cstring_length(
|
|
1462
|
+
source_id->valuestring, (size_t)limits->max_string_bytes,
|
|
1463
|
+
runtime, &source_len, error);
|
|
1464
|
+
if (code != TF_TRANSFORM_OK || source_len == 0)
|
|
1465
|
+
return code != TF_TRANSFORM_OK ? code : tf_transform_set_error(
|
|
1466
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe sourceId");
|
|
1467
|
+
out->source_id = (char *)tf_transform_resource_malloc(
|
|
1468
|
+
ledger, source_len + 1, error);
|
|
1469
|
+
if (!out->source_id) return ledger->last_code != TF_TRANSFORM_OK
|
|
1470
|
+
? ledger->last_code : tf_transform_set_error(
|
|
1471
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
1472
|
+
"recipe sourceId allocation failed");
|
|
1473
|
+
code = tf_transform_copy_bytes_runtime(
|
|
1474
|
+
out->source_id, source_id->valuestring, source_len + 1,
|
|
1475
|
+
runtime, error);
|
|
1476
|
+
if (code != TF_TRANSFORM_OK) goto cleanup;
|
|
1477
|
+
out->source_id_len = source_len;
|
|
1478
|
+
numeric = required_item(value, "numeric");
|
|
1479
|
+
categorical = required_item(value, "categorical");
|
|
1480
|
+
if (has_numeric) {
|
|
1481
|
+
if (!parse_numeric_recipe_branch(numeric, out)) goto invalid;
|
|
1482
|
+
} else if (!cJSON_IsNull(numeric)) goto invalid;
|
|
1483
|
+
if (has_categorical) {
|
|
1484
|
+
if (!parse_categorical_recipe_branch(categorical, out)) goto invalid;
|
|
1485
|
+
const cJSON *categories = required_item(required_item(categorical, "encode"), "categories");
|
|
1486
|
+
if (cJSON_IsArray(categories)) {
|
|
1487
|
+
code = parse_fixed_categories(categories, runtime, ledger, out, error);
|
|
1488
|
+
if (code != TF_TRANSFORM_OK) goto cleanup;
|
|
1489
|
+
}
|
|
1490
|
+
} else if (!cJSON_IsNull(categorical)) goto invalid;
|
|
1491
|
+
return TF_TRANSFORM_OK;
|
|
1492
|
+
invalid:
|
|
1493
|
+
code = tf_transform_set_error(
|
|
1494
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1495
|
+
"invalid prepared-transform recipe column");
|
|
1496
|
+
cleanup:
|
|
1497
|
+
free(out->source_id);
|
|
1498
|
+
free(out->categorical_fixed);
|
|
1499
|
+
tf_transform_resource_release(ledger,
|
|
1500
|
+
(uint64_t)out->categorical_fixed_count * sizeof(*out->categorical_fixed));
|
|
1501
|
+
tf_transform_resource_release(ledger, (uint64_t)source_len + 1);
|
|
1502
|
+
memset(out, 0, sizeof(*out));
|
|
1503
|
+
return code;
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
static tf_transform_code recipe_from_json_value(
|
|
1507
|
+
const cJSON *root, const tf_transform_runtime_copy *runtime,
|
|
1508
|
+
tf_transform_resource_ledger *ledger,
|
|
1509
|
+
tf_transform_recipe **out, tf_transform_error **error) {
|
|
1510
|
+
static const char *const top_keys[] = {
|
|
1511
|
+
"columns", "format", "outputDtype", "policyVersion",
|
|
1512
|
+
"semanticLimits", "version"
|
|
1513
|
+
};
|
|
1514
|
+
static const char *const semantic_keys[] = {
|
|
1515
|
+
"maxOutputColumns", "maxOutputElementsPerApply"
|
|
1516
|
+
};
|
|
1517
|
+
const tf_transform_limits_v1 *limits = &runtime->limits;
|
|
1518
|
+
const cJSON *columns;
|
|
1519
|
+
const cJSON *semantic;
|
|
1520
|
+
const cJSON *column_json;
|
|
1521
|
+
tf_transform_recipe *recipe = NULL;
|
|
1522
|
+
tf_transform_code code;
|
|
1523
|
+
uint64_t version;
|
|
1524
|
+
uint64_t policy_version;
|
|
1525
|
+
uint64_t max_columns;
|
|
1526
|
+
uint64_t max_elements;
|
|
1527
|
+
size_t count = 0;
|
|
1528
|
+
if (out) *out = NULL;
|
|
1529
|
+
if (!root || !runtime || !ledger || !out)
|
|
1530
|
+
return tf_transform_set_error(
|
|
1531
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
1532
|
+
"recipe value argument is null");
|
|
1533
|
+
if (!exact_keys(root, top_keys, 6)
|
|
1534
|
+
|| !json_string_equals(required_item(root, "format"),
|
|
1535
|
+
"tranfi.transform-recipe")
|
|
1536
|
+
|| !json_string_equals(required_item(root, "outputDtype"), "float64")
|
|
1537
|
+
|| !safe_uint64(required_item(root, "version"), &version)
|
|
1538
|
+
|| !safe_uint64(required_item(root, "policyVersion"), &policy_version))
|
|
1539
|
+
return tf_transform_set_error(
|
|
1540
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe header");
|
|
1541
|
+
if (version != 1 || policy_version != 1)
|
|
1542
|
+
return tf_transform_set_error(
|
|
1543
|
+
error, TF_TRANSFORM_UNSUPPORTED_VERSION,
|
|
1544
|
+
"recipe or policy version is unsupported");
|
|
1545
|
+
semantic = required_item(root, "semanticLimits");
|
|
1546
|
+
if (!exact_keys(semantic, semantic_keys, 2)
|
|
1547
|
+
|| !safe_uint64(required_item(semantic, "maxOutputColumns"), &max_columns)
|
|
1548
|
+
|| !safe_uint64(required_item(semantic, "maxOutputElementsPerApply"),
|
|
1549
|
+
&max_elements)
|
|
1550
|
+
|| max_columns == 0 || max_elements == 0)
|
|
1551
|
+
return tf_transform_set_error(
|
|
1552
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1553
|
+
"invalid recipe semantic limits");
|
|
1554
|
+
if (max_columns > limits->max_output_columns
|
|
1555
|
+
|| max_elements > limits->max_output_elements_per_call)
|
|
1556
|
+
return tf_transform_set_error(
|
|
1557
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1558
|
+
"recipe semantic limits exceed host limits");
|
|
1559
|
+
columns = required_item(root, "columns");
|
|
1560
|
+
if (!cJSON_IsArray(columns))
|
|
1561
|
+
return tf_transform_set_error(
|
|
1562
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe columns");
|
|
1563
|
+
cJSON_ArrayForEach(column_json, columns) {
|
|
1564
|
+
if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
1565
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
1566
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1567
|
+
}
|
|
1568
|
+
if (count == SIZE_MAX) return tf_transform_set_error(
|
|
1569
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1570
|
+
"recipe column count overflows");
|
|
1571
|
+
++count;
|
|
1572
|
+
}
|
|
1573
|
+
if (count == 0 || (uint64_t)count > limits->max_steps
|
|
1574
|
+
|| (uint64_t)count > limits->max_input_columns)
|
|
1575
|
+
return tf_transform_set_error(
|
|
1576
|
+
error, TF_TRANSFORM_INVALID_RECIPE, "invalid recipe columns");
|
|
1577
|
+
if ((uint64_t)count > max_columns)
|
|
1578
|
+
return tf_transform_set_error(
|
|
1579
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1580
|
+
"recipe output columns exceed its semantic limit");
|
|
1581
|
+
if (count > SIZE_MAX / sizeof(tf_transform_recipe_column))
|
|
1582
|
+
return tf_transform_set_error(
|
|
1583
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
1584
|
+
"recipe column allocation overflows");
|
|
1585
|
+
recipe = (tf_transform_recipe *)tf_transform_resource_calloc(
|
|
1586
|
+
ledger, 1, sizeof(*recipe), error);
|
|
1587
|
+
if (!recipe) return ledger->last_code != TF_TRANSFORM_OK
|
|
1588
|
+
? ledger->last_code : tf_transform_set_error(
|
|
1589
|
+
error, TF_TRANSFORM_ALLOCATION, "recipe allocation failed");
|
|
1590
|
+
atomic_init(&recipe->refcount, 1u);
|
|
1591
|
+
recipe->column_count = count;
|
|
1592
|
+
recipe->max_output_columns = max_columns;
|
|
1593
|
+
recipe->max_output_elements_per_apply = max_elements;
|
|
1594
|
+
recipe->columns = (tf_transform_recipe_column *)
|
|
1595
|
+
tf_transform_resource_calloc(
|
|
1596
|
+
ledger, count, sizeof(*recipe->columns), error);
|
|
1597
|
+
if (!recipe->columns) {
|
|
1598
|
+
code = ledger->last_code != TF_TRANSFORM_OK
|
|
1599
|
+
? ledger->last_code : tf_transform_set_error(
|
|
1600
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
1601
|
+
"recipe column allocation failed");
|
|
1602
|
+
goto done;
|
|
1603
|
+
}
|
|
1604
|
+
column_json = columns->child;
|
|
1605
|
+
for (size_t i = 0; i < count; ++i, column_json = column_json->next) {
|
|
1606
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
1607
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
1608
|
+
code = parse_recipe_column(
|
|
1609
|
+
column_json, runtime, ledger, &recipe->columns[i], error);
|
|
1610
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
1611
|
+
for (size_t j = 0; j < i; ++j) {
|
|
1612
|
+
int equal = 0;
|
|
1613
|
+
code = runtime_span_equal(
|
|
1614
|
+
recipe->columns[i].source_id,
|
|
1615
|
+
recipe->columns[i].source_id_len,
|
|
1616
|
+
recipe->columns[j].source_id,
|
|
1617
|
+
recipe->columns[j].source_id_len,
|
|
1618
|
+
runtime, &equal, error);
|
|
1619
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
1620
|
+
if (equal) {
|
|
1621
|
+
code = tf_transform_set_error(
|
|
1622
|
+
error, TF_TRANSFORM_INVALID_RECIPE,
|
|
1623
|
+
"recipe sourceIds must be unique");
|
|
1624
|
+
goto done;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
*out = recipe;
|
|
1629
|
+
recipe = NULL;
|
|
1630
|
+
code = TF_TRANSFORM_OK;
|
|
1631
|
+
done:
|
|
1632
|
+
tf_transform_recipe_release(recipe);
|
|
1633
|
+
return code;
|
|
1634
|
+
}
|
|
1635
|
+
|
|
1636
|
+
tf_transform_code tf_transform_recipe_parse_numeric(
|
|
1637
|
+
const uint8_t *json, size_t json_len,
|
|
1638
|
+
const tf_transform_limits_v1 *limits,
|
|
1639
|
+
tf_transform_recipe **out, tf_transform_error **error) {
|
|
1640
|
+
cJSON *root = NULL;
|
|
1641
|
+
tf_transform_runtime_copy runtime;
|
|
1642
|
+
tf_transform_resource_ledger ledger;
|
|
1643
|
+
tf_transform_code code;
|
|
1644
|
+
uint64_t ast_resident = 0;
|
|
1645
|
+
if (out) *out = NULL;
|
|
1646
|
+
if (!json || json_len == 0 || !limits || !out)
|
|
1647
|
+
return tf_transform_set_error(
|
|
1648
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "recipe argument is null or empty");
|
|
1649
|
+
if (json_len > limits->max_recipe_bytes)
|
|
1650
|
+
return tf_transform_set_error(
|
|
1651
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "recipe exceeds byte limit");
|
|
1652
|
+
memset(&runtime, 0, sizeof(runtime));
|
|
1653
|
+
runtime.limits = *limits;
|
|
1654
|
+
tf_transform_resource_ledger_init(&ledger, &runtime);
|
|
1655
|
+
code = tf_transform_json_parse_bounded_runtime_ledger(
|
|
1656
|
+
json, json_len, &runtime, &ledger, &root,
|
|
1657
|
+
&ast_resident, NULL, error, TF_TRANSFORM_INVALID_RECIPE);
|
|
1658
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1659
|
+
code = recipe_from_json_value(root, &runtime, &ledger, out, error);
|
|
1660
|
+
cJSON_Delete(root);
|
|
1661
|
+
tf_transform_resource_release(&ledger, ast_resident);
|
|
1662
|
+
return code;
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
tf_transform_code tf_transform_recipe_from_json(
|
|
1666
|
+
const uint8_t *json, size_t json_len,
|
|
1667
|
+
const tf_transform_limits_v1 *limits,
|
|
1668
|
+
tf_transform_recipe **out, tf_transform_error **error) {
|
|
1669
|
+
tf_transform_limits_v1 copied;
|
|
1670
|
+
tf_transform_code code;
|
|
1671
|
+
if (out) *out = NULL;
|
|
1672
|
+
tf_transform_clear_error(error);
|
|
1673
|
+
if (!out) return tf_transform_set_error(
|
|
1674
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "recipe output is null");
|
|
1675
|
+
code = tf_transform_copy_limits(limits, &copied, error);
|
|
1676
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
1677
|
+
return tf_transform_recipe_parse_numeric(json, json_len, &copied, out, error);
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
static cJSON *json_null(void) { return cJSON_CreateNull(); }
|
|
1681
|
+
static cJSON *json_string(const char *value) { return cJSON_CreateString(value); }
|
|
1682
|
+
static cJSON *json_number(double value) { return cJSON_CreateNumber(value); }
|
|
1683
|
+
|
|
1684
|
+
static int add_item(cJSON *object, const char *key, cJSON *item) {
|
|
1685
|
+
if (!item) return 0;
|
|
1686
|
+
if (!cJSON_AddItemToObject(object, key, item)) {
|
|
1687
|
+
cJSON_Delete(item);
|
|
1688
|
+
return 0;
|
|
1689
|
+
}
|
|
1690
|
+
return 1;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
static int transfer_item(cJSON *object, const char *key, cJSON **item) {
|
|
1694
|
+
cJSON *owned;
|
|
1695
|
+
if (!item) return 0;
|
|
1696
|
+
owned = *item;
|
|
1697
|
+
*item = NULL;
|
|
1698
|
+
return add_item(object, key, owned);
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
static int transfer_array_item(cJSON *array, cJSON **item) {
|
|
1702
|
+
cJSON *owned;
|
|
1703
|
+
if (!item || !*item) return 0;
|
|
1704
|
+
owned = *item;
|
|
1705
|
+
*item = NULL;
|
|
1706
|
+
if (!cJSON_AddItemToArray(array, owned)) {
|
|
1707
|
+
cJSON_Delete(owned);
|
|
1708
|
+
return 0;
|
|
1709
|
+
}
|
|
1710
|
+
return 1;
|
|
1711
|
+
}
|
|
1712
|
+
|
|
1713
|
+
static cJSON *tagged_bits(uint64_t raw, uint32_t dtype) {
|
|
1714
|
+
static const char hex[] = "0123456789abcdef";
|
|
1715
|
+
cJSON *object = cJSON_CreateObject();
|
|
1716
|
+
char bits[17];
|
|
1717
|
+
size_t digits;
|
|
1718
|
+
if (!object) return NULL;
|
|
1719
|
+
if (dtype == TF_VIEW_FLOAT32) {
|
|
1720
|
+
raw &= UINT64_C(0xffffffff);
|
|
1721
|
+
digits = 8;
|
|
1722
|
+
if (!add_item(object, "t", json_string("f32"))) goto fail;
|
|
1723
|
+
} else if (dtype == TF_VIEW_FLOAT64) {
|
|
1724
|
+
digits = 16;
|
|
1725
|
+
if (!add_item(object, "t", json_string("f64"))) goto fail;
|
|
1726
|
+
} else goto fail;
|
|
1727
|
+
for (size_t i = 0; i < digits; ++i) {
|
|
1728
|
+
unsigned int shift = (unsigned int)((digits - 1 - i) * 4);
|
|
1729
|
+
bits[i] = hex[(raw >> shift) & 15u];
|
|
1730
|
+
}
|
|
1731
|
+
bits[digits] = '\0';
|
|
1732
|
+
if (!add_item(object, "v", json_string(bits))) goto fail;
|
|
1733
|
+
return object;
|
|
1734
|
+
fail:
|
|
1735
|
+
cJSON_Delete(object);
|
|
1736
|
+
return NULL;
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
static cJSON *tagged_constant(double value, uint32_t dtype) {
|
|
1740
|
+
if (dtype == TF_VIEW_FLOAT32) {
|
|
1741
|
+
float narrowed = (float)value;
|
|
1742
|
+
uint32_t raw;
|
|
1743
|
+
memcpy(&raw, &narrowed, sizeof(raw));
|
|
1744
|
+
return tagged_bits(raw, dtype);
|
|
1745
|
+
}
|
|
1746
|
+
return tagged_bits(tf_transform_double_bits(value), dtype);
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
typedef struct tf_json_build_budget {
|
|
1750
|
+
uint64_t nodes;
|
|
1751
|
+
uint64_t strings;
|
|
1752
|
+
uint64_t string_bytes;
|
|
1753
|
+
uint64_t object_keys;
|
|
1754
|
+
uint64_t objects;
|
|
1755
|
+
uint64_t max_string_bytes;
|
|
1756
|
+
uint64_t max_object_width;
|
|
1757
|
+
uint64_t depth;
|
|
1758
|
+
} tf_json_build_budget;
|
|
1759
|
+
|
|
1760
|
+
static int build_budget_add(uint64_t *target, uint64_t value) {
|
|
1761
|
+
if (*target > UINT64_MAX - value) return 0;
|
|
1762
|
+
*target += value;
|
|
1763
|
+
return 1;
|
|
1764
|
+
}
|
|
1765
|
+
|
|
1766
|
+
static int build_budget_node(tf_json_build_budget *budget) {
|
|
1767
|
+
return build_budget_add(&budget->nodes, 1);
|
|
1768
|
+
}
|
|
1769
|
+
|
|
1770
|
+
static int build_budget_string(
|
|
1771
|
+
tf_json_build_budget *budget, uint64_t bytes) {
|
|
1772
|
+
if (!build_budget_add(&budget->strings, 1)
|
|
1773
|
+
|| !build_budget_add(&budget->string_bytes, bytes)) return 0;
|
|
1774
|
+
if (bytes > budget->max_string_bytes) budget->max_string_bytes = bytes;
|
|
1775
|
+
return 1;
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
static int build_budget_key(
|
|
1779
|
+
tf_json_build_budget *budget, const char *key) {
|
|
1780
|
+
size_t len = strlen(key);
|
|
1781
|
+
return build_budget_add(&budget->object_keys, 1)
|
|
1782
|
+
&& build_budget_string(budget, (uint64_t)len);
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
static int build_budget_object(
|
|
1786
|
+
tf_json_build_budget *budget, uint64_t width) {
|
|
1787
|
+
if (!build_budget_node(budget)
|
|
1788
|
+
|| !build_budget_add(&budget->objects, 1)) return 0;
|
|
1789
|
+
if (width > budget->max_object_width) budget->max_object_width = width;
|
|
1790
|
+
return 1;
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1793
|
+
static int build_budget_array(tf_json_build_budget *budget) {
|
|
1794
|
+
return build_budget_node(budget);
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
static int build_budget_primitive(
|
|
1798
|
+
tf_json_build_budget *budget, const char *key) {
|
|
1799
|
+
return build_budget_key(budget, key) && build_budget_node(budget);
|
|
1800
|
+
}
|
|
1801
|
+
|
|
1802
|
+
static int build_budget_string_property(
|
|
1803
|
+
tf_json_build_budget *budget, const char *key, uint64_t value_bytes) {
|
|
1804
|
+
return build_budget_primitive(budget, key)
|
|
1805
|
+
&& build_budget_string(budget, value_bytes);
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
static int build_budget_child(
|
|
1809
|
+
tf_json_build_budget *budget, const char *key) {
|
|
1810
|
+
return build_budget_key(budget, key);
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
static int build_budget_tagged(
|
|
1814
|
+
tf_json_build_budget *budget, const char *key, uint64_t value_bytes) {
|
|
1815
|
+
return build_budget_child(budget, key)
|
|
1816
|
+
&& build_budget_object(budget, 2)
|
|
1817
|
+
&& build_budget_string_property(budget, "t", 3)
|
|
1818
|
+
&& build_budget_string_property(budget, "v", value_bytes);
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
static int build_budget_schema(
|
|
1822
|
+
const tf_transform_schema *schema, tf_json_build_budget *budget) {
|
|
1823
|
+
if (!schema || !build_budget_array(budget)) return 0;
|
|
1824
|
+
for (size_t i = 0; i < schema->field_count; ++i) {
|
|
1825
|
+
const tf_transform_schema_field_owned *field = &schema->fields[i];
|
|
1826
|
+
if (!build_budget_object(budget, schema->is_output ? 6 : 3)) return 0;
|
|
1827
|
+
if (schema->is_output) {
|
|
1828
|
+
if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_NONE) {
|
|
1829
|
+
if (!build_budget_primitive(budget, "category")) return 0;
|
|
1830
|
+
} else if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_OTHER) {
|
|
1831
|
+
if (!build_budget_child(budget, "category")
|
|
1832
|
+
|| !build_budget_object(budget, 1)
|
|
1833
|
+
|| !build_budget_string_property(budget, "t", 5)) return 0;
|
|
1834
|
+
} else {
|
|
1835
|
+
if (!build_budget_tagged(
|
|
1836
|
+
budget, "category",
|
|
1837
|
+
field->category_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
if (!build_budget_string_property(budget, "dtype", 7)
|
|
1841
|
+
|| !build_budget_string_property(
|
|
1842
|
+
budget, "id", (uint64_t)field->id_len)
|
|
1843
|
+
|| !build_budget_string_property(
|
|
1844
|
+
budget, "name", (uint64_t)field->name_len)) return 0;
|
|
1845
|
+
if (schema->is_output
|
|
1846
|
+
&& (!build_budget_string_property(
|
|
1847
|
+
budget, "role",
|
|
1848
|
+
field->role == TF_TRANSFORM_ROLE_ONEHOT ? 6 : 5)
|
|
1849
|
+
|| !build_budget_string_property(
|
|
1850
|
+
budget, "sourceId", (uint64_t)field->source_id_len))) return 0;
|
|
1851
|
+
}
|
|
1852
|
+
if (budget->depth < 3) budget->depth = 3;
|
|
1853
|
+
return 1;
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1856
|
+
static int build_budget_numeric_recipe_branch(
|
|
1857
|
+
const tf_transform_recipe_column *column, tf_json_build_budget *budget) {
|
|
1858
|
+
uint64_t impute_bytes = column->impute == TF_TRANSFORM_IMPUTE_CONSTANT ? 8
|
|
1859
|
+
: column->impute == TF_TRANSFORM_IMPUTE_MEDIAN ? 6 : 4;
|
|
1860
|
+
uint64_t normalize_bytes
|
|
1861
|
+
= column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD ? 8
|
|
1862
|
+
: column->normalize == TF_TRANSFORM_NORMALIZE_MINMAX ? 6 : 4;
|
|
1863
|
+
if (!build_budget_child(budget, "numeric")
|
|
1864
|
+
|| !build_budget_object(budget, 2)
|
|
1865
|
+
|| !build_budget_child(budget, "impute")
|
|
1866
|
+
|| !build_budget_object(budget, 3)) return 0;
|
|
1867
|
+
if (column->impute == TF_TRANSFORM_IMPUTE_MEAN
|
|
1868
|
+
|| column->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
|
|
1869
|
+
if (!build_budget_string_property(
|
|
1870
|
+
budget, "allMissing",
|
|
1871
|
+
column->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO ? 4 : 5))
|
|
1872
|
+
return 0;
|
|
1873
|
+
} else if (!build_budget_primitive(budget, "allMissing")) return 0;
|
|
1874
|
+
if (column->impute == TF_TRANSFORM_IMPUTE_CONSTANT) {
|
|
1875
|
+
if (!build_budget_tagged(
|
|
1876
|
+
budget, "constant",
|
|
1877
|
+
column->constant_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
|
|
1878
|
+
} else if (!build_budget_primitive(budget, "constant")) return 0;
|
|
1879
|
+
return build_budget_string_property(budget, "op", impute_bytes)
|
|
1880
|
+
&& build_budget_child(budget, "normalize")
|
|
1881
|
+
&& build_budget_object(budget, 2)
|
|
1882
|
+
&& build_budget_primitive(budget, "ddof")
|
|
1883
|
+
&& build_budget_string_property(budget, "op", normalize_bytes);
|
|
1884
|
+
}
|
|
1885
|
+
|
|
1886
|
+
static int build_budget_categorical_recipe_branch(
|
|
1887
|
+
const tf_transform_recipe_column *column, tf_json_build_budget *budget) {
|
|
1888
|
+
if (!build_budget_child(budget, "categorical")
|
|
1889
|
+
|| !build_budget_object(budget, 2)
|
|
1890
|
+
|| !build_budget_child(budget, "encode")
|
|
1891
|
+
|| !build_budget_object(budget, 4)) return 0;
|
|
1892
|
+
if (column->categorical_fixed_count) {
|
|
1893
|
+
if (!build_budget_child(budget, "categories") || !build_budget_array(budget)) return 0;
|
|
1894
|
+
for (size_t i = 0; i < column->categorical_fixed_count; ++i)
|
|
1895
|
+
if (!build_budget_object(budget, 2)
|
|
1896
|
+
|| !build_budget_string_property(budget, "t", 3)
|
|
1897
|
+
|| !build_budget_string_property(budget, "v",
|
|
1898
|
+
column->categorical_fixed_dtype == TF_VIEW_FLOAT32 ? 8 : 16)) return 0;
|
|
1899
|
+
if (budget->depth < 9) budget->depth = 9;
|
|
1900
|
+
} else if (column->categorical_discover
|
|
1901
|
+
? !build_budget_string_property(budget, "categories", 8)
|
|
1902
|
+
: !build_budget_primitive(budget, "categories")) return 0;
|
|
1903
|
+
if (!build_budget_string_property(
|
|
1904
|
+
budget, "op",
|
|
1905
|
+
column->categorical_encode == TF_TRANSFORM_ENCODE_LABEL ? 5
|
|
1906
|
+
: column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT ? 6 : 4)
|
|
1907
|
+
|| !build_budget_primitive(budget, "sentinelLabel")) return 0;
|
|
1908
|
+
if (column->categorical_unknown == TF_TRANSFORM_UNKNOWN_NONE) {
|
|
1909
|
+
if (!build_budget_primitive(budget, "unknown")) return 0;
|
|
1910
|
+
} else if (!build_budget_string_property(
|
|
1911
|
+
budget, "unknown",
|
|
1912
|
+
column->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? 8
|
|
1913
|
+
: column->categorical_unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO ? 8 : 5))
|
|
1914
|
+
return 0;
|
|
1915
|
+
if (!build_budget_child(budget, "impute")
|
|
1916
|
+
|| !build_budget_object(budget, 3)) return 0;
|
|
1917
|
+
if (column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
|
|
1918
|
+
if (!build_budget_primitive(budget, "allMissing")) return 0;
|
|
1919
|
+
} else if (!build_budget_string_property(
|
|
1920
|
+
budget, "allMissing",
|
|
1921
|
+
column->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO ? 4 : 5))
|
|
1922
|
+
return 0;
|
|
1923
|
+
return build_budget_primitive(budget, "constant")
|
|
1924
|
+
&& build_budget_string_property(budget, "op", 4);
|
|
1925
|
+
}
|
|
1926
|
+
|
|
1927
|
+
static int build_budget_recipe(
|
|
1928
|
+
const tf_transform_recipe *recipe, tf_json_build_budget *budget) {
|
|
1929
|
+
if (!recipe || !build_budget_object(budget, 6)
|
|
1930
|
+
|| !build_budget_child(budget, "columns")
|
|
1931
|
+
|| !build_budget_array(budget)) return 0;
|
|
1932
|
+
for (size_t i = 0; i < recipe->column_count; ++i) {
|
|
1933
|
+
const tf_transform_recipe_column *column = &recipe->columns[i];
|
|
1934
|
+
if (!build_budget_object(budget, 4)
|
|
1935
|
+
|| !build_budget_child(budget, "kind")
|
|
1936
|
+
|| !build_budget_object(budget, 4)
|
|
1937
|
+
|| (column->kind == TF_TRANSFORM_KIND_INFER
|
|
1938
|
+
? (!build_budget_primitive(budget, "maxCategories")
|
|
1939
|
+
|| !build_budget_string_property(budget, "op", 5)
|
|
1940
|
+
|| !build_budget_string_property(budget, "rule", 29)
|
|
1941
|
+
|| !build_budget_primitive(budget, "value"))
|
|
1942
|
+
: (!build_budget_primitive(budget, "maxCategories")
|
|
1943
|
+
|| !build_budget_string_property(budget, "op", 8)
|
|
1944
|
+
|| !build_budget_primitive(budget, "rule")
|
|
1945
|
+
|| !build_budget_string_property(
|
|
1946
|
+
budget, "value",
|
|
1947
|
+
column->kind == TF_TRANSFORM_KIND_CATEGORICAL ? 11 : 7))))
|
|
1948
|
+
return 0;
|
|
1949
|
+
if (column->kind == TF_TRANSFORM_KIND_NUMERIC) {
|
|
1950
|
+
if (!build_budget_primitive(budget, "categorical")
|
|
1951
|
+
|| !build_budget_numeric_recipe_branch(column, budget)) return 0;
|
|
1952
|
+
} else if (column->kind == TF_TRANSFORM_KIND_CATEGORICAL) {
|
|
1953
|
+
if (!build_budget_categorical_recipe_branch(column, budget)
|
|
1954
|
+
|| !build_budget_primitive(budget, "numeric")) return 0;
|
|
1955
|
+
} else if (column->kind == TF_TRANSFORM_KIND_INFER) {
|
|
1956
|
+
if (!build_budget_categorical_recipe_branch(column, budget)
|
|
1957
|
+
|| !build_budget_numeric_recipe_branch(column, budget)) return 0;
|
|
1958
|
+
} else return 0;
|
|
1959
|
+
if (!build_budget_string_property(
|
|
1960
|
+
budget, "sourceId", (uint64_t)column->source_id_len)) return 0;
|
|
1961
|
+
}
|
|
1962
|
+
if (!build_budget_string_property(budget, "format", 23)
|
|
1963
|
+
|| !build_budget_string_property(budget, "outputDtype", 7)
|
|
1964
|
+
|| !build_budget_primitive(budget, "policyVersion")
|
|
1965
|
+
|| !build_budget_child(budget, "semanticLimits")
|
|
1966
|
+
|| !build_budget_object(budget, 2)
|
|
1967
|
+
|| !build_budget_primitive(budget, "maxOutputColumns")
|
|
1968
|
+
|| !build_budget_primitive(budget, "maxOutputElementsPerApply")
|
|
1969
|
+
|| !build_budget_primitive(budget, "version")) return 0;
|
|
1970
|
+
if (budget->depth < 8) budget->depth = 8;
|
|
1971
|
+
return 1;
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
static int build_budget_step(
|
|
1975
|
+
const tf_transform_recipe_column *recipe,
|
|
1976
|
+
const tf_transform_column_state *column_state,
|
|
1977
|
+
tf_json_build_budget *budget) {
|
|
1978
|
+
if (column_state->kind == TF_TRANSFORM_KIND_CATEGORICAL) {
|
|
1979
|
+
const tf_transform_categorical_state *state
|
|
1980
|
+
= &column_state->value.categorical;
|
|
1981
|
+
uint64_t digits = state->source_dtype == TF_VIEW_FLOAT32 ? 8 : 16;
|
|
1982
|
+
if (column_state->kind != TF_TRANSFORM_KIND_CATEGORICAL
|
|
1983
|
+
|| !build_budget_object(budget, 6)
|
|
1984
|
+
|| !build_budget_child(budget, "categorical")
|
|
1985
|
+
|| !build_budget_object(budget, 3)
|
|
1986
|
+
|| !build_budget_child(budget, "categories")
|
|
1987
|
+
|| !build_budget_array(budget)) return 0;
|
|
1988
|
+
for (size_t i = 0; i < state->category_count; ++i)
|
|
1989
|
+
if (!build_budget_object(budget, 2)
|
|
1990
|
+
|| !build_budget_string_property(budget, "t", 3)
|
|
1991
|
+
|| !build_budget_string_property(budget, "v", digits)) return 0;
|
|
1992
|
+
if (!build_budget_child(budget, "encode")
|
|
1993
|
+
|| !build_budget_object(budget, 4)
|
|
1994
|
+
|| !build_budget_string_property(
|
|
1995
|
+
budget, "op",
|
|
1996
|
+
state->encode == TF_TRANSFORM_ENCODE_LABEL ? 5
|
|
1997
|
+
: state->encode == TF_TRANSFORM_ENCODE_ONEHOT ? 6 : 4)
|
|
1998
|
+
|| !build_budget_primitive(budget, "otherOrdinal")
|
|
1999
|
+
|| !build_budget_primitive(budget, "sentinelLabel")) return 0;
|
|
2000
|
+
if (state->unknown == TF_TRANSFORM_UNKNOWN_NONE) {
|
|
2001
|
+
if (!build_budget_primitive(budget, "unknown")) return 0;
|
|
2002
|
+
} else if (!build_budget_string_property(
|
|
2003
|
+
budget, "unknown",
|
|
2004
|
+
state->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? 8
|
|
2005
|
+
: state->unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO ? 8 : 5))
|
|
2006
|
+
return 0;
|
|
2007
|
+
if (!build_budget_child(budget, "impute")
|
|
2008
|
+
|| !build_budget_object(budget, 3)) return 0;
|
|
2009
|
+
if (state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
|
|
2010
|
+
if (!build_budget_primitive(budget, "allMissing")
|
|
2011
|
+
|| !build_budget_string_property(budget, "op", 4)
|
|
2012
|
+
|| !build_budget_primitive(budget, "value")) return 0;
|
|
2013
|
+
} else if (!build_budget_string_property(budget, "allMissing", 5)
|
|
2014
|
+
|| !build_budget_string_property(budget, "op", 4)
|
|
2015
|
+
|| !build_budget_tagged(budget, "value", digits)) return 0;
|
|
2016
|
+
if (!build_budget_string_property(budget, "kind", 11)
|
|
2017
|
+
|| !build_budget_primitive(budget, "numeric")
|
|
2018
|
+
|| !build_budget_string_property(
|
|
2019
|
+
budget, "sourceId", (uint64_t)recipe->source_id_len)
|
|
2020
|
+
|| !build_budget_primitive(budget, "stateVersion")) return 0;
|
|
2021
|
+
return 1;
|
|
2022
|
+
}
|
|
2023
|
+
{
|
|
2024
|
+
const tf_transform_numeric_state *state = &column_state->value.numeric;
|
|
2025
|
+
if (column_state->kind != TF_TRANSFORM_KIND_NUMERIC) return 0;
|
|
2026
|
+
if (!build_budget_object(budget, 6)
|
|
2027
|
+
|| !build_budget_primitive(budget, "categorical")
|
|
2028
|
+
|| !build_budget_string_property(budget, "kind", 7)
|
|
2029
|
+
|| !build_budget_child(budget, "numeric")
|
|
2030
|
+
|| !build_budget_object(budget, 2)
|
|
2031
|
+
|| !build_budget_child(budget, "impute")
|
|
2032
|
+
|| !build_budget_object(budget, 3)) return 0;
|
|
2033
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_MEAN
|
|
2034
|
+
|| recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
|
|
2035
|
+
if (!build_budget_string_property(
|
|
2036
|
+
budget, "allMissing", 5)) return 0;
|
|
2037
|
+
} else if (!build_budget_primitive(budget, "allMissing")) return 0;
|
|
2038
|
+
if (!build_budget_string_property(budget, "op", 8)) return 0;
|
|
2039
|
+
if (state->has_impute_value) {
|
|
2040
|
+
if (!build_budget_tagged(budget, "value", 16)) return 0;
|
|
2041
|
+
} else if (!build_budget_primitive(budget, "value")) return 0;
|
|
2042
|
+
if (!build_budget_child(budget, "normalize")
|
|
2043
|
+
|| !build_budget_object(budget, 5)
|
|
2044
|
+
|| !build_budget_primitive(budget, "ddof")
|
|
2045
|
+
|| !build_budget_tagged(budget, "location", 16)
|
|
2046
|
+
|| !build_budget_string_property(budget, "op", 8)
|
|
2047
|
+
|| !build_budget_tagged(budget, "scale", 16)
|
|
2048
|
+
|| !build_budget_string_property(
|
|
2049
|
+
budget, "sourceId", (uint64_t)recipe->source_id_len)
|
|
2050
|
+
|| !build_budget_primitive(budget, "stateVersion")) return 0;
|
|
2051
|
+
return 1;
|
|
2052
|
+
}
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2055
|
+
static tf_transform_code check_build_budget(
|
|
2056
|
+
const tf_json_build_budget *budget,
|
|
2057
|
+
const tf_transform_limits_v1 *limits,
|
|
2058
|
+
uint64_t extra_allocations, uint64_t output_copies,
|
|
2059
|
+
tf_transform_error **error) {
|
|
2060
|
+
uint64_t allocations = budget->nodes;
|
|
2061
|
+
uint64_t resident;
|
|
2062
|
+
uint64_t output_upper;
|
|
2063
|
+
uint64_t sort_bytes;
|
|
2064
|
+
if (!build_budget_add(&allocations, budget->strings)
|
|
2065
|
+
|| !build_budget_add(&allocations, budget->objects)
|
|
2066
|
+
|| !build_budget_add(&allocations, extra_allocations)) goto overflow;
|
|
2067
|
+
if (budget->nodes > UINT64_MAX / sizeof(cJSON)) goto overflow;
|
|
2068
|
+
resident = budget->nodes * sizeof(cJSON);
|
|
2069
|
+
if (!build_budget_add(&resident, budget->string_bytes)
|
|
2070
|
+
|| !build_budget_add(&resident, budget->strings)) goto overflow;
|
|
2071
|
+
if (budget->max_object_width > UINT64_MAX / sizeof(cJSON *)) goto overflow;
|
|
2072
|
+
sort_bytes = budget->max_object_width * sizeof(cJSON *);
|
|
2073
|
+
if (!build_budget_add(&resident, sort_bytes)) goto overflow;
|
|
2074
|
+
if (budget->nodes > UINT64_MAX / 32) goto overflow;
|
|
2075
|
+
output_upper = budget->nodes * 32;
|
|
2076
|
+
if (budget->string_bytes > (UINT64_MAX - output_upper) / 6)
|
|
2077
|
+
goto overflow;
|
|
2078
|
+
output_upper += budget->string_bytes * 6;
|
|
2079
|
+
if (output_upper > limits->max_plan_bytes)
|
|
2080
|
+
output_upper = limits->max_plan_bytes;
|
|
2081
|
+
if (output_copies != 0 && output_upper > UINT64_MAX / output_copies)
|
|
2082
|
+
goto overflow;
|
|
2083
|
+
if (!build_budget_add(&resident, output_upper * output_copies)) goto overflow;
|
|
2084
|
+
if (budget->depth > limits->max_json_depth
|
|
2085
|
+
|| budget->object_keys > limits->max_object_keys
|
|
2086
|
+
|| budget->string_bytes > limits->max_decoded_string_bytes
|
|
2087
|
+
|| budget->max_string_bytes > limits->max_string_bytes
|
|
2088
|
+
|| allocations > limits->max_allocations_per_session
|
|
2089
|
+
|| resident > limits->max_resident_state_bytes
|
|
2090
|
+
|| sizeof(cJSON) > limits->max_allocation_bytes
|
|
2091
|
+
|| sort_bytes > limits->max_allocation_bytes
|
|
2092
|
+
|| (budget->strings != 0
|
|
2093
|
+
&& (budget->max_string_bytes == UINT64_MAX
|
|
2094
|
+
|| budget->max_string_bytes + 1
|
|
2095
|
+
> limits->max_allocation_bytes)))
|
|
2096
|
+
return tf_transform_set_error(
|
|
2097
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2098
|
+
"JSON construction exceeds configured limits");
|
|
2099
|
+
return TF_TRANSFORM_OK;
|
|
2100
|
+
overflow:
|
|
2101
|
+
return tf_transform_set_error(
|
|
2102
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2103
|
+
"JSON construction resource estimate overflows");
|
|
2104
|
+
}
|
|
2105
|
+
|
|
2106
|
+
tf_transform_code tf_transform_schema_json_preflight(
|
|
2107
|
+
const tf_transform_schema *schema, const tf_transform_limits_v1 *limits,
|
|
2108
|
+
tf_transform_error **error) {
|
|
2109
|
+
tf_json_build_budget budget = {0};
|
|
2110
|
+
if (!schema || !limits)
|
|
2111
|
+
return tf_transform_set_error(
|
|
2112
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2113
|
+
"schema JSON preflight argument is null");
|
|
2114
|
+
if ((uint64_t)schema->field_count
|
|
2115
|
+
> (schema->is_output ? limits->max_output_columns
|
|
2116
|
+
: limits->max_input_columns)
|
|
2117
|
+
|| !build_budget_schema(schema, &budget))
|
|
2118
|
+
return tf_transform_set_error(
|
|
2119
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2120
|
+
"schema JSON construction estimate overflows");
|
|
2121
|
+
return check_build_budget(&budget, limits, 1, 1, error);
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
tf_transform_code tf_transform_plan_json_preflight(
|
|
2125
|
+
const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
|
|
2126
|
+
tf_transform_error **error) {
|
|
2127
|
+
tf_json_build_budget fingerprint = {0};
|
|
2128
|
+
tf_json_build_budget plan_json = {0};
|
|
2129
|
+
tf_transform_code code;
|
|
2130
|
+
uint64_t total_allocations;
|
|
2131
|
+
uint64_t total_categories = 0;
|
|
2132
|
+
if (!plan || !limits)
|
|
2133
|
+
return tf_transform_set_error(
|
|
2134
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2135
|
+
"plan JSON preflight argument is null");
|
|
2136
|
+
if ((uint64_t)plan->input_schema.field_count > limits->max_input_columns
|
|
2137
|
+
|| (uint64_t)plan->output_schema.field_count > limits->max_output_columns
|
|
2138
|
+
|| (uint64_t)plan->input_schema.field_count > limits->max_steps)
|
|
2139
|
+
return tf_transform_set_error(
|
|
2140
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2141
|
+
"plan JSON structure exceeds configured limits");
|
|
2142
|
+
for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
|
|
2143
|
+
uint64_t count;
|
|
2144
|
+
if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
|
|
2145
|
+
&& (plan->recipe->columns[i].infer_max_categories
|
|
2146
|
+
> limits->max_categories_per_column
|
|
2147
|
+
|| plan->recipe->columns[i].infer_max_categories
|
|
2148
|
+
> limits->max_total_categories))
|
|
2149
|
+
return tf_transform_set_error(
|
|
2150
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2151
|
+
"inferred category bound exceeds configured limits");
|
|
2152
|
+
if (plan->states[i].kind != TF_TRANSFORM_KIND_CATEGORICAL) continue;
|
|
2153
|
+
count = (uint64_t)plan->states[i].value.categorical.category_count;
|
|
2154
|
+
if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
|
|
2155
|
+
&& (count < 2
|
|
2156
|
+
|| count > plan->recipe->columns[i].infer_max_categories))
|
|
2157
|
+
return tf_transform_set_error(
|
|
2158
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
2159
|
+
"inferred categorical plan state is inconsistent");
|
|
2160
|
+
if (count > limits->max_categories_per_column
|
|
2161
|
+
|| total_categories > UINT64_MAX - count
|
|
2162
|
+
|| total_categories + count > limits->max_total_categories)
|
|
2163
|
+
return tf_transform_set_error(
|
|
2164
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2165
|
+
"plan categories exceed configured limits");
|
|
2166
|
+
total_categories += count;
|
|
2167
|
+
}
|
|
2168
|
+
if (!build_budget_object(&fingerprint, 3)
|
|
2169
|
+
|| !build_budget_child(&fingerprint, "inputSchema")
|
|
2170
|
+
|| !build_budget_schema(&plan->input_schema, &fingerprint)
|
|
2171
|
+
|| !build_budget_primitive(&fingerprint, "policyVersion")
|
|
2172
|
+
|| !build_budget_child(&fingerprint, "recipe")
|
|
2173
|
+
|| !build_budget_recipe(plan->recipe, &fingerprint)) goto overflow;
|
|
2174
|
+
if (!build_budget_object(&plan_json, 7)
|
|
2175
|
+
|| !build_budget_string_property(&plan_json, "format", 21)
|
|
2176
|
+
|| !build_budget_child(&plan_json, "inputSchema")
|
|
2177
|
+
|| !build_budget_schema(&plan->input_schema, &plan_json)
|
|
2178
|
+
|| !build_budget_child(&plan_json, "outputSchema")
|
|
2179
|
+
|| !build_budget_schema(&plan->output_schema, &plan_json)
|
|
2180
|
+
|| !build_budget_child(&plan_json, "recipe")
|
|
2181
|
+
|| !build_budget_recipe(plan->recipe, &plan_json)
|
|
2182
|
+
|| !build_budget_string_property(&plan_json, "recipeSha256", 64)
|
|
2183
|
+
|| !build_budget_child(&plan_json, "steps")
|
|
2184
|
+
|| !build_budget_array(&plan_json)) goto overflow;
|
|
2185
|
+
for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
|
|
2186
|
+
if (!build_budget_step(
|
|
2187
|
+
&plan->recipe->columns[i], &plan->states[i], &plan_json))
|
|
2188
|
+
goto overflow;
|
|
2189
|
+
}
|
|
2190
|
+
if (!build_budget_primitive(&plan_json, "version")) goto overflow;
|
|
2191
|
+
if (plan_json.depth < 8) plan_json.depth = 8;
|
|
2192
|
+
code = check_build_budget(&fingerprint, limits, 1, 1, error);
|
|
2193
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2194
|
+
code = check_build_budget(&plan_json, limits, 2, 2, error);
|
|
2195
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2196
|
+
total_allocations = fingerprint.nodes;
|
|
2197
|
+
if (!build_budget_add(&total_allocations, fingerprint.strings)
|
|
2198
|
+
|| !build_budget_add(&total_allocations, fingerprint.objects)
|
|
2199
|
+
|| !build_budget_add(&total_allocations, plan_json.nodes)
|
|
2200
|
+
|| !build_budget_add(&total_allocations, plan_json.strings)
|
|
2201
|
+
|| !build_budget_add(&total_allocations, plan_json.objects)
|
|
2202
|
+
|| !build_budget_add(&total_allocations, 3)) goto overflow;
|
|
2203
|
+
if (total_allocations > limits->max_allocations_per_session)
|
|
2204
|
+
return tf_transform_set_error(
|
|
2205
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2206
|
+
"plan JSON allocation count exceeds configured limit");
|
|
2207
|
+
return TF_TRANSFORM_OK;
|
|
2208
|
+
overflow:
|
|
2209
|
+
return tf_transform_set_error(
|
|
2210
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2211
|
+
"plan JSON construction estimate overflows");
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
static cJSON *recipe_numeric_branch_to_json(
|
|
2215
|
+
const tf_transform_recipe_column *column) {
|
|
2216
|
+
cJSON *numeric = cJSON_CreateObject();
|
|
2217
|
+
cJSON *impute = cJSON_CreateObject();
|
|
2218
|
+
cJSON *normalize = cJSON_CreateObject();
|
|
2219
|
+
const char *impute_name;
|
|
2220
|
+
const char *normalize_name;
|
|
2221
|
+
if (!numeric || !impute || !normalize) goto fail;
|
|
2222
|
+
if (column->impute == TF_TRANSFORM_IMPUTE_MEAN
|
|
2223
|
+
|| column->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
|
|
2224
|
+
if (!add_item(impute, "allMissing", json_string(
|
|
2225
|
+
column->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
|
|
2226
|
+
? "zero" : "error"))) goto fail;
|
|
2227
|
+
} else if (!add_item(impute, "allMissing", json_null())) goto fail;
|
|
2228
|
+
if (column->impute == TF_TRANSFORM_IMPUTE_CONSTANT) {
|
|
2229
|
+
if (!add_item(impute, "constant", tagged_constant(
|
|
2230
|
+
column->constant, column->constant_dtype))) goto fail;
|
|
2231
|
+
} else if (!add_item(impute, "constant", json_null())) goto fail;
|
|
2232
|
+
impute_name = column->impute == TF_TRANSFORM_IMPUTE_NONE ? "none"
|
|
2233
|
+
: column->impute == TF_TRANSFORM_IMPUTE_ZERO ? "zero"
|
|
2234
|
+
: column->impute == TF_TRANSFORM_IMPUTE_CONSTANT ? "constant"
|
|
2235
|
+
: column->impute == TF_TRANSFORM_IMPUTE_MEAN ? "mean" : "median";
|
|
2236
|
+
if (!add_item(impute, "op", json_string(impute_name))
|
|
2237
|
+
|| !transfer_item(numeric, "impute", &impute)) goto fail;
|
|
2238
|
+
if (column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
|
|
2239
|
+
if (!add_item(normalize, "ddof", json_number(column->ddof))) goto fail;
|
|
2240
|
+
} else if (!add_item(normalize, "ddof", json_null())) goto fail;
|
|
2241
|
+
normalize_name = column->normalize == TF_TRANSFORM_NORMALIZE_NONE ? "none"
|
|
2242
|
+
: column->normalize == TF_TRANSFORM_NORMALIZE_STANDARD
|
|
2243
|
+
? "standard" : "minmax";
|
|
2244
|
+
if (!add_item(normalize, "op", json_string(normalize_name))
|
|
2245
|
+
|| !transfer_item(numeric, "normalize", &normalize)) goto fail;
|
|
2246
|
+
return numeric;
|
|
2247
|
+
fail:
|
|
2248
|
+
cJSON_Delete(numeric);
|
|
2249
|
+
cJSON_Delete(impute);
|
|
2250
|
+
cJSON_Delete(normalize);
|
|
2251
|
+
return NULL;
|
|
2252
|
+
}
|
|
2253
|
+
|
|
2254
|
+
static cJSON *recipe_categories_to_json(const tf_transform_recipe_column *column) {
|
|
2255
|
+
if (!column->categorical_fixed_count)
|
|
2256
|
+
return column->categorical_discover ? json_string("discover") : json_null();
|
|
2257
|
+
cJSON *categories = cJSON_CreateArray();
|
|
2258
|
+
if (!categories) return NULL;
|
|
2259
|
+
for (size_t i = 0; i < column->categorical_fixed_count; ++i) {
|
|
2260
|
+
cJSON *value = tagged_bits(column->categorical_fixed[i].bits,
|
|
2261
|
+
column->categorical_fixed_dtype);
|
|
2262
|
+
if (!value || !transfer_array_item(categories, &value)) {
|
|
2263
|
+
cJSON_Delete(value);
|
|
2264
|
+
cJSON_Delete(categories);
|
|
2265
|
+
return NULL;
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
return categories;
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
static cJSON *recipe_categorical_branch_to_json(
|
|
2272
|
+
const tf_transform_recipe_column *column) {
|
|
2273
|
+
cJSON *categorical = cJSON_CreateObject();
|
|
2274
|
+
cJSON *impute = cJSON_CreateObject();
|
|
2275
|
+
cJSON *encode = cJSON_CreateObject();
|
|
2276
|
+
cJSON *sentinel = column->categorical_has_sentinel_label
|
|
2277
|
+
? json_number((double)column->categorical_sentinel_label) : json_null();
|
|
2278
|
+
cJSON *unknown = column->categorical_unknown == TF_TRANSFORM_UNKNOWN_NONE
|
|
2279
|
+
? json_null() : json_string(
|
|
2280
|
+
column->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL
|
|
2281
|
+
? "sentinel"
|
|
2282
|
+
: column->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER
|
|
2283
|
+
? "other"
|
|
2284
|
+
: column->categorical_unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO
|
|
2285
|
+
? "all_zero" : "error");
|
|
2286
|
+
if (!categorical || !impute || !encode || !sentinel || !unknown
|
|
2287
|
+
|| !add_item(encode, "categories", recipe_categories_to_json(column))
|
|
2288
|
+
|| !add_item(encode, "op", json_string(
|
|
2289
|
+
column->categorical_encode == TF_TRANSFORM_ENCODE_LABEL ? "label"
|
|
2290
|
+
: column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT
|
|
2291
|
+
? "onehot" : "none"))
|
|
2292
|
+
|| !transfer_item(encode, "sentinelLabel", &sentinel)
|
|
2293
|
+
|| !transfer_item(encode, "unknown", &unknown)
|
|
2294
|
+
|| !transfer_item(categorical, "encode", &encode)
|
|
2295
|
+
|| !add_item(impute, "allMissing",
|
|
2296
|
+
column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
2297
|
+
? json_null() : json_string(
|
|
2298
|
+
column->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
|
|
2299
|
+
? "zero" : "error"))
|
|
2300
|
+
|| !add_item(impute, "constant", json_null())
|
|
2301
|
+
|| !add_item(impute, "op", json_string(
|
|
2302
|
+
column->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
2303
|
+
? "none" : "mode"))
|
|
2304
|
+
|| !transfer_item(categorical, "impute", &impute)) goto fail;
|
|
2305
|
+
return categorical;
|
|
2306
|
+
fail:
|
|
2307
|
+
cJSON_Delete(categorical);
|
|
2308
|
+
cJSON_Delete(impute);
|
|
2309
|
+
cJSON_Delete(encode);
|
|
2310
|
+
cJSON_Delete(sentinel);
|
|
2311
|
+
cJSON_Delete(unknown);
|
|
2312
|
+
return NULL;
|
|
2313
|
+
}
|
|
2314
|
+
|
|
2315
|
+
cJSON *tf_transform_recipe_to_json(const tf_transform_recipe *recipe) {
|
|
2316
|
+
cJSON *root = cJSON_CreateObject();
|
|
2317
|
+
cJSON *columns = cJSON_CreateArray();
|
|
2318
|
+
cJSON *semantic = NULL;
|
|
2319
|
+
if (!recipe || !root || !columns) goto fail;
|
|
2320
|
+
if (!transfer_item(root, "columns", &columns)) goto fail;
|
|
2321
|
+
for (size_t i = 0; i < recipe->column_count; ++i) {
|
|
2322
|
+
const tf_transform_recipe_column *column = &recipe->columns[i];
|
|
2323
|
+
cJSON *entry = cJSON_CreateObject();
|
|
2324
|
+
cJSON *kind = cJSON_CreateObject();
|
|
2325
|
+
cJSON *numeric = column->kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
2326
|
+
? json_null() : recipe_numeric_branch_to_json(column);
|
|
2327
|
+
cJSON *categorical = column->kind == TF_TRANSFORM_KIND_NUMERIC
|
|
2328
|
+
? json_null() : recipe_categorical_branch_to_json(column);
|
|
2329
|
+
cJSON *max_categories = column->kind == TF_TRANSFORM_KIND_INFER
|
|
2330
|
+
? json_number((double)column->infer_max_categories) : json_null();
|
|
2331
|
+
cJSON *rule = column->kind == TF_TRANSFORM_KIND_INFER
|
|
2332
|
+
? json_string("finite-integer-cardinality-v1") : json_null();
|
|
2333
|
+
cJSON *kind_value = column->kind == TF_TRANSFORM_KIND_INFER
|
|
2334
|
+
? json_null() : json_string(
|
|
2335
|
+
column->kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
2336
|
+
? "categorical" : "numeric");
|
|
2337
|
+
if (!entry || !kind || !numeric || !categorical || !max_categories
|
|
2338
|
+
|| !rule || !kind_value
|
|
2339
|
+
|| !transfer_item(entry, "categorical", &categorical)
|
|
2340
|
+
|| !transfer_item(kind, "maxCategories", &max_categories)
|
|
2341
|
+
|| !add_item(kind, "op", json_string(
|
|
2342
|
+
column->kind == TF_TRANSFORM_KIND_INFER ? "infer" : "declared"))
|
|
2343
|
+
|| !transfer_item(kind, "rule", &rule)
|
|
2344
|
+
|| !transfer_item(kind, "value", &kind_value)
|
|
2345
|
+
|| !transfer_item(entry, "kind", &kind)
|
|
2346
|
+
|| !transfer_item(entry, "numeric", &numeric)
|
|
2347
|
+
|| !add_item(entry, "sourceId", json_string(column->source_id))
|
|
2348
|
+
|| !transfer_array_item(
|
|
2349
|
+
cJSON_GetObjectItemCaseSensitive(root, "columns"), &entry)) {
|
|
2350
|
+
cJSON_Delete(entry);
|
|
2351
|
+
cJSON_Delete(kind);
|
|
2352
|
+
cJSON_Delete(numeric);
|
|
2353
|
+
cJSON_Delete(categorical);
|
|
2354
|
+
cJSON_Delete(max_categories);
|
|
2355
|
+
cJSON_Delete(rule);
|
|
2356
|
+
cJSON_Delete(kind_value);
|
|
2357
|
+
goto fail;
|
|
2358
|
+
}
|
|
2359
|
+
continue;
|
|
2360
|
+
}
|
|
2361
|
+
if (!add_item(root, "format", json_string("tranfi.transform-recipe"))
|
|
2362
|
+
|| !add_item(root, "outputDtype", json_string("float64"))
|
|
2363
|
+
|| !add_item(root, "policyVersion", json_number(1))) goto fail;
|
|
2364
|
+
semantic = cJSON_CreateObject();
|
|
2365
|
+
if (!semantic
|
|
2366
|
+
|| !add_item(semantic, "maxOutputColumns",
|
|
2367
|
+
json_number((double)recipe->max_output_columns))
|
|
2368
|
+
|| !add_item(semantic, "maxOutputElementsPerApply",
|
|
2369
|
+
json_number((double)recipe->max_output_elements_per_apply)))
|
|
2370
|
+
goto fail;
|
|
2371
|
+
if (!transfer_item(root, "semanticLimits", &semantic)) goto fail;
|
|
2372
|
+
if (!add_item(root, "version", json_number(1))) goto fail;
|
|
2373
|
+
return root;
|
|
2374
|
+
fail:
|
|
2375
|
+
cJSON_Delete(columns);
|
|
2376
|
+
cJSON_Delete(semantic);
|
|
2377
|
+
cJSON_Delete(root);
|
|
2378
|
+
return NULL;
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
cJSON *tf_transform_schema_to_json(const tf_transform_schema *schema) {
|
|
2382
|
+
cJSON *array = cJSON_CreateArray();
|
|
2383
|
+
if (!schema || !array) return NULL;
|
|
2384
|
+
for (size_t i = 0; i < schema->field_count; ++i) {
|
|
2385
|
+
const tf_transform_schema_field_owned *field = &schema->fields[i];
|
|
2386
|
+
cJSON *entry = cJSON_CreateObject();
|
|
2387
|
+
if (!entry) goto fail;
|
|
2388
|
+
if (schema->is_output) {
|
|
2389
|
+
cJSON *category = NULL;
|
|
2390
|
+
if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_NONE)
|
|
2391
|
+
category = json_null();
|
|
2392
|
+
else if (field->category_kind == TF_TRANSFORM_SCHEMA_CATEGORY_OTHER) {
|
|
2393
|
+
category = cJSON_CreateObject();
|
|
2394
|
+
if (category && !add_item(category, "t", json_string("other"))) {
|
|
2395
|
+
cJSON_Delete(category);
|
|
2396
|
+
category = NULL;
|
|
2397
|
+
}
|
|
2398
|
+
} else category = tagged_bits(
|
|
2399
|
+
field->category_bits, field->category_dtype);
|
|
2400
|
+
if (!category || !transfer_item(entry, "category", &category)) {
|
|
2401
|
+
cJSON_Delete(category);
|
|
2402
|
+
goto entry_fail;
|
|
2403
|
+
}
|
|
2404
|
+
}
|
|
2405
|
+
if (!add_item(entry, "dtype", json_string(
|
|
2406
|
+
field->dtype == TF_VIEW_FLOAT32 ? "float32" : "float64"))
|
|
2407
|
+
|| !add_item(entry, "id", json_string(field->id))
|
|
2408
|
+
|| !add_item(entry, "name", json_string(field->name))) goto entry_fail;
|
|
2409
|
+
if (schema->is_output
|
|
2410
|
+
&& (!add_item(entry, "role", json_string(
|
|
2411
|
+
field->role == TF_TRANSFORM_ROLE_LABEL ? "label"
|
|
2412
|
+
: field->role == TF_TRANSFORM_ROLE_ONEHOT ? "onehot" : "value"))
|
|
2413
|
+
|| !add_item(entry, "sourceId", json_string(field->source_id))))
|
|
2414
|
+
goto entry_fail;
|
|
2415
|
+
if (!transfer_array_item(array, &entry)) goto entry_fail;
|
|
2416
|
+
continue;
|
|
2417
|
+
entry_fail:
|
|
2418
|
+
cJSON_Delete(entry);
|
|
2419
|
+
goto fail;
|
|
2420
|
+
}
|
|
2421
|
+
return array;
|
|
2422
|
+
fail:
|
|
2423
|
+
cJSON_Delete(array);
|
|
2424
|
+
return NULL;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
static void digest_hex(const uint8_t digest[32], char out[65]) {
|
|
2428
|
+
static const char digits[] = "0123456789abcdef";
|
|
2429
|
+
for (size_t i = 0; i < 32; ++i) {
|
|
2430
|
+
out[i * 2] = digits[digest[i] >> 4];
|
|
2431
|
+
out[i * 2 + 1] = digits[digest[i] & 15u];
|
|
2432
|
+
}
|
|
2433
|
+
out[64] = '\0';
|
|
2434
|
+
}
|
|
2435
|
+
|
|
2436
|
+
tf_transform_code tf_transform_plan_recipe_fingerprint(
|
|
2437
|
+
const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
|
|
2438
|
+
char out[65], tf_transform_error **error) {
|
|
2439
|
+
cJSON *object = NULL;
|
|
2440
|
+
cJSON *schema = NULL;
|
|
2441
|
+
cJSON *recipe_json = NULL;
|
|
2442
|
+
uint8_t *canonical = NULL;
|
|
2443
|
+
size_t canonical_len = 0;
|
|
2444
|
+
uint8_t digest[32];
|
|
2445
|
+
tf_transform_code code;
|
|
2446
|
+
if (!plan || !limits || !out) return tf_transform_set_error(
|
|
2447
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2448
|
+
"recipe fingerprint argument is null");
|
|
2449
|
+
object = cJSON_CreateObject();
|
|
2450
|
+
if (!object) return tf_transform_set_error(
|
|
2451
|
+
error, TF_TRANSFORM_ALLOCATION, "recipe fingerprint allocation failed");
|
|
2452
|
+
schema = tf_transform_schema_to_json(&plan->input_schema);
|
|
2453
|
+
recipe_json = tf_transform_recipe_to_json(plan->recipe);
|
|
2454
|
+
if (!schema || !recipe_json
|
|
2455
|
+
|| !transfer_item(object, "inputSchema", &schema)) goto allocation_failed;
|
|
2456
|
+
if (!add_item(object, "policyVersion", json_number(1))
|
|
2457
|
+
|| !transfer_item(object, "recipe", &recipe_json)) goto allocation_failed;
|
|
2458
|
+
code = tf_transform_json_print_canonical(
|
|
2459
|
+
object, limits, &canonical, &canonical_len, error);
|
|
2460
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2461
|
+
tf_transform_sha256(canonical, canonical_len, digest);
|
|
2462
|
+
digest_hex(digest, out);
|
|
2463
|
+
done:
|
|
2464
|
+
tf_transform_bytes_free(&canonical, &canonical_len);
|
|
2465
|
+
cJSON_Delete(schema);
|
|
2466
|
+
cJSON_Delete(recipe_json);
|
|
2467
|
+
cJSON_Delete(object);
|
|
2468
|
+
return code;
|
|
2469
|
+
allocation_failed:
|
|
2470
|
+
code = tf_transform_set_error(
|
|
2471
|
+
error, TF_TRANSFORM_ALLOCATION, "recipe fingerprint JSON failed");
|
|
2472
|
+
goto done;
|
|
2473
|
+
}
|
|
2474
|
+
|
|
2475
|
+
static tf_transform_code imported_recipe_fingerprint(
|
|
2476
|
+
const cJSON *plan_root, const tf_transform_runtime_copy *runtime,
|
|
2477
|
+
tf_transform_resource_ledger *ledger, char out[65],
|
|
2478
|
+
tf_transform_error **error) {
|
|
2479
|
+
const cJSON *input_schema = required_item(plan_root, "inputSchema");
|
|
2480
|
+
const cJSON *recipe = required_item(plan_root, "recipe");
|
|
2481
|
+
cJSON object;
|
|
2482
|
+
cJSON input_copy;
|
|
2483
|
+
cJSON policy;
|
|
2484
|
+
cJSON recipe_copy;
|
|
2485
|
+
uint8_t *canonical = NULL;
|
|
2486
|
+
size_t canonical_len = 0;
|
|
2487
|
+
uint8_t digest[32];
|
|
2488
|
+
tf_transform_code code;
|
|
2489
|
+
if (!input_schema || !recipe)
|
|
2490
|
+
return tf_transform_set_error(
|
|
2491
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2492
|
+
"recipe fingerprint inputs are missing");
|
|
2493
|
+
memset(&object, 0, sizeof(object));
|
|
2494
|
+
input_copy = *input_schema;
|
|
2495
|
+
memset(&policy, 0, sizeof(policy));
|
|
2496
|
+
recipe_copy = *recipe;
|
|
2497
|
+
object.type = cJSON_Object;
|
|
2498
|
+
object.child = &input_copy;
|
|
2499
|
+
input_copy.string = (char *)"inputSchema";
|
|
2500
|
+
input_copy.prev = NULL;
|
|
2501
|
+
input_copy.next = &policy;
|
|
2502
|
+
policy.type = cJSON_Number;
|
|
2503
|
+
policy.valueint = 1;
|
|
2504
|
+
policy.valuedouble = 1.0;
|
|
2505
|
+
policy.string = (char *)"policyVersion";
|
|
2506
|
+
policy.prev = &input_copy;
|
|
2507
|
+
policy.next = &recipe_copy;
|
|
2508
|
+
recipe_copy.string = (char *)"recipe";
|
|
2509
|
+
recipe_copy.prev = &policy;
|
|
2510
|
+
recipe_copy.next = NULL;
|
|
2511
|
+
code = tf_transform_json_print_canonical_runtime_ledger(
|
|
2512
|
+
&object, runtime, ledger, &canonical, &canonical_len, error);
|
|
2513
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2514
|
+
code = tf_transform_sha256_runtime(
|
|
2515
|
+
canonical, canonical_len, runtime, digest, error);
|
|
2516
|
+
if (code == TF_TRANSFORM_OK) digest_hex(digest, out);
|
|
2517
|
+
free(canonical);
|
|
2518
|
+
tf_transform_resource_release(ledger, (uint64_t)canonical_len);
|
|
2519
|
+
return code;
|
|
2520
|
+
}
|
|
2521
|
+
|
|
2522
|
+
static const char *impute_op_name(tf_transform_impute_op op) {
|
|
2523
|
+
if (op == TF_TRANSFORM_IMPUTE_NONE) return "none";
|
|
2524
|
+
if (op == TF_TRANSFORM_IMPUTE_ZERO) return "zero";
|
|
2525
|
+
if (op == TF_TRANSFORM_IMPUTE_CONSTANT) return "constant";
|
|
2526
|
+
if (op == TF_TRANSFORM_IMPUTE_MEAN) return "mean";
|
|
2527
|
+
return "median";
|
|
2528
|
+
}
|
|
2529
|
+
|
|
2530
|
+
static const char *normalize_op_name(tf_transform_normalize_op op) {
|
|
2531
|
+
if (op == TF_TRANSFORM_NORMALIZE_NONE) return "none";
|
|
2532
|
+
if (op == TF_TRANSFORM_NORMALIZE_STANDARD) return "standard";
|
|
2533
|
+
return "minmax";
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
static cJSON *numeric_step_to_json(
|
|
2537
|
+
const tf_transform_recipe_column *recipe,
|
|
2538
|
+
const tf_transform_numeric_state *state) {
|
|
2539
|
+
cJSON *step = cJSON_CreateObject();
|
|
2540
|
+
cJSON *numeric = cJSON_CreateObject();
|
|
2541
|
+
cJSON *impute = cJSON_CreateObject();
|
|
2542
|
+
cJSON *normalize = cJSON_CreateObject();
|
|
2543
|
+
if (!step || !numeric || !impute || !normalize) goto fail;
|
|
2544
|
+
if (!add_item(step, "categorical", json_null())
|
|
2545
|
+
|| !add_item(step, "kind", json_string("numeric"))) goto fail;
|
|
2546
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_MEAN
|
|
2547
|
+
|| recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN) {
|
|
2548
|
+
if (!add_item(impute, "allMissing", json_string(
|
|
2549
|
+
recipe->all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
|
|
2550
|
+
? "zero" : "error"))) goto fail;
|
|
2551
|
+
} else if (!add_item(impute, "allMissing", json_null())) goto fail;
|
|
2552
|
+
if (!add_item(impute, "op", json_string(impute_op_name(recipe->impute))))
|
|
2553
|
+
goto fail;
|
|
2554
|
+
if (state->has_impute_value) {
|
|
2555
|
+
if (!add_item(impute, "value", tagged_constant(
|
|
2556
|
+
state->impute_value, TF_VIEW_FLOAT64))) goto fail;
|
|
2557
|
+
} else if (!add_item(impute, "value", json_null())) goto fail;
|
|
2558
|
+
if (!transfer_item(numeric, "impute", &impute)) goto fail;
|
|
2559
|
+
if (recipe->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
|
|
2560
|
+
if (!add_item(normalize, "ddof", json_number(recipe->ddof))) goto fail;
|
|
2561
|
+
} else if (!add_item(normalize, "ddof", json_null())) goto fail;
|
|
2562
|
+
if (!add_item(normalize, "location", tagged_constant(
|
|
2563
|
+
state->location, TF_VIEW_FLOAT64))
|
|
2564
|
+
|| !add_item(normalize, "op", json_string(
|
|
2565
|
+
normalize_op_name(recipe->normalize)))
|
|
2566
|
+
|| !add_item(normalize, "scale", tagged_constant(
|
|
2567
|
+
state->scale, TF_VIEW_FLOAT64))) goto fail;
|
|
2568
|
+
if (!transfer_item(numeric, "normalize", &normalize)) goto fail;
|
|
2569
|
+
if (!transfer_item(step, "numeric", &numeric)) goto fail;
|
|
2570
|
+
if (!add_item(step, "sourceId", json_string(recipe->source_id))
|
|
2571
|
+
|| !add_item(step, "stateVersion", json_number(1))) goto fail;
|
|
2572
|
+
return step;
|
|
2573
|
+
fail:
|
|
2574
|
+
cJSON_Delete(step);
|
|
2575
|
+
cJSON_Delete(numeric);
|
|
2576
|
+
cJSON_Delete(impute);
|
|
2577
|
+
cJSON_Delete(normalize);
|
|
2578
|
+
return NULL;
|
|
2579
|
+
}
|
|
2580
|
+
|
|
2581
|
+
static cJSON *categorical_step_to_json(
|
|
2582
|
+
const tf_transform_recipe_column *recipe,
|
|
2583
|
+
const tf_transform_categorical_state *state) {
|
|
2584
|
+
cJSON *step = cJSON_CreateObject();
|
|
2585
|
+
cJSON *categorical = cJSON_CreateObject();
|
|
2586
|
+
cJSON *categories = cJSON_CreateArray();
|
|
2587
|
+
cJSON *encode = cJSON_CreateObject();
|
|
2588
|
+
cJSON *impute = cJSON_CreateObject();
|
|
2589
|
+
cJSON *other_ordinal = state->has_other_ordinal
|
|
2590
|
+
? json_number((double)state->other_ordinal) : json_null();
|
|
2591
|
+
cJSON *sentinel = state->has_sentinel_label
|
|
2592
|
+
? json_number((double)state->sentinel_label) : json_null();
|
|
2593
|
+
cJSON *unknown = state->unknown == TF_TRANSFORM_UNKNOWN_NONE
|
|
2594
|
+
? json_null() : json_string(
|
|
2595
|
+
state->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL ? "sentinel"
|
|
2596
|
+
: state->unknown == TF_TRANSFORM_UNKNOWN_OTHER ? "other"
|
|
2597
|
+
: state->unknown == TF_TRANSFORM_UNKNOWN_ALL_ZERO
|
|
2598
|
+
? "all_zero" : "error");
|
|
2599
|
+
cJSON *impute_all_missing
|
|
2600
|
+
= state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
2601
|
+
? json_null() : json_string(
|
|
2602
|
+
recipe->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
|
|
2603
|
+
? "zero" : "error");
|
|
2604
|
+
cJSON *impute_value
|
|
2605
|
+
= state->has_impute_value
|
|
2606
|
+
? tagged_bits(state->impute_bits, state->source_dtype) : json_null();
|
|
2607
|
+
if (!step || !categorical || !categories || !encode || !impute
|
|
2608
|
+
|| !other_ordinal || !sentinel || !unknown
|
|
2609
|
+
|| !impute_all_missing || !impute_value) goto fail;
|
|
2610
|
+
for (size_t i = 0; i < state->category_count; ++i) {
|
|
2611
|
+
cJSON *tagged = tagged_bits(
|
|
2612
|
+
state->categories[i].bits, state->source_dtype);
|
|
2613
|
+
if (!tagged || !transfer_array_item(categories, &tagged)) {
|
|
2614
|
+
cJSON_Delete(tagged);
|
|
2615
|
+
goto fail;
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
2618
|
+
if (!transfer_item(categorical, "categories", &categories)
|
|
2619
|
+
|| !add_item(encode, "op", json_string(
|
|
2620
|
+
state->encode == TF_TRANSFORM_ENCODE_LABEL ? "label"
|
|
2621
|
+
: state->encode == TF_TRANSFORM_ENCODE_ONEHOT ? "onehot" : "none"))
|
|
2622
|
+
|| !transfer_item(encode, "otherOrdinal", &other_ordinal)
|
|
2623
|
+
|| !transfer_item(encode, "sentinelLabel", &sentinel)
|
|
2624
|
+
|| !transfer_item(encode, "unknown", &unknown)
|
|
2625
|
+
|| !transfer_item(categorical, "encode", &encode)
|
|
2626
|
+
|| !transfer_item(impute, "allMissing", &impute_all_missing)
|
|
2627
|
+
|| !add_item(impute, "op", json_string(
|
|
2628
|
+
state->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
2629
|
+
? "none" : "mode"))
|
|
2630
|
+
|| !transfer_item(impute, "value", &impute_value)
|
|
2631
|
+
|| !transfer_item(categorical, "impute", &impute)
|
|
2632
|
+
|| !transfer_item(step, "categorical", &categorical)
|
|
2633
|
+
|| !add_item(step, "kind", json_string("categorical"))
|
|
2634
|
+
|| !add_item(step, "numeric", json_null())
|
|
2635
|
+
|| !add_item(step, "sourceId", json_string(recipe->source_id))
|
|
2636
|
+
|| !add_item(step, "stateVersion", json_number(1))) goto fail;
|
|
2637
|
+
return step;
|
|
2638
|
+
fail:
|
|
2639
|
+
cJSON_Delete(step);
|
|
2640
|
+
cJSON_Delete(categorical);
|
|
2641
|
+
cJSON_Delete(categories);
|
|
2642
|
+
cJSON_Delete(encode);
|
|
2643
|
+
cJSON_Delete(impute);
|
|
2644
|
+
cJSON_Delete(other_ordinal);
|
|
2645
|
+
cJSON_Delete(sentinel);
|
|
2646
|
+
cJSON_Delete(unknown);
|
|
2647
|
+
cJSON_Delete(impute_all_missing);
|
|
2648
|
+
cJSON_Delete(impute_value);
|
|
2649
|
+
return NULL;
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
cJSON *tf_transform_plan_to_json(
|
|
2653
|
+
const tf_transform_plan *plan, const tf_transform_limits_v1 *limits,
|
|
2654
|
+
tf_transform_error **error) {
|
|
2655
|
+
cJSON *root = NULL;
|
|
2656
|
+
cJSON *input_schema = NULL;
|
|
2657
|
+
cJSON *output_schema = NULL;
|
|
2658
|
+
cJSON *recipe = NULL;
|
|
2659
|
+
cJSON *steps = NULL;
|
|
2660
|
+
char fingerprint[65];
|
|
2661
|
+
tf_transform_code code;
|
|
2662
|
+
if (!plan || !limits) {
|
|
2663
|
+
tf_transform_set_error(
|
|
2664
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2665
|
+
"plan JSON argument is null");
|
|
2666
|
+
goto failed;
|
|
2667
|
+
}
|
|
2668
|
+
code = tf_transform_plan_json_preflight(plan, limits, error);
|
|
2669
|
+
if (code != TF_TRANSFORM_OK) goto failed;
|
|
2670
|
+
root = cJSON_CreateObject();
|
|
2671
|
+
steps = cJSON_CreateArray();
|
|
2672
|
+
if (!root || !steps) goto allocation_failed;
|
|
2673
|
+
code = tf_transform_plan_recipe_fingerprint(
|
|
2674
|
+
plan, limits, fingerprint, error);
|
|
2675
|
+
if (code != TF_TRANSFORM_OK) goto failed;
|
|
2676
|
+
input_schema = tf_transform_schema_to_json(&plan->input_schema);
|
|
2677
|
+
output_schema = tf_transform_schema_to_json(&plan->output_schema);
|
|
2678
|
+
recipe = tf_transform_recipe_to_json(plan->recipe);
|
|
2679
|
+
if (!input_schema || !output_schema || !recipe
|
|
2680
|
+
|| !add_item(root, "format", json_string("tranfi.transform-plan"))
|
|
2681
|
+
|| !transfer_item(root, "inputSchema", &input_schema)) goto allocation_failed;
|
|
2682
|
+
if (!transfer_item(root, "outputSchema", &output_schema)) goto allocation_failed;
|
|
2683
|
+
if (!transfer_item(root, "recipe", &recipe)) goto allocation_failed;
|
|
2684
|
+
if (!add_item(root, "recipeSha256", json_string(fingerprint)))
|
|
2685
|
+
goto allocation_failed;
|
|
2686
|
+
for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
|
|
2687
|
+
cJSON *step = plan->states[i].kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
2688
|
+
? categorical_step_to_json(
|
|
2689
|
+
&plan->recipe->columns[i],
|
|
2690
|
+
&plan->states[i].value.categorical)
|
|
2691
|
+
: numeric_step_to_json(
|
|
2692
|
+
&plan->recipe->columns[i],
|
|
2693
|
+
&plan->states[i].value.numeric);
|
|
2694
|
+
if (!step || !transfer_array_item(steps, &step)) {
|
|
2695
|
+
cJSON_Delete(step);
|
|
2696
|
+
goto allocation_failed;
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
if (!transfer_item(root, "steps", &steps)) goto allocation_failed;
|
|
2700
|
+
if (!add_item(root, "version", json_number(1))) goto allocation_failed;
|
|
2701
|
+
return root;
|
|
2702
|
+
allocation_failed:
|
|
2703
|
+
tf_transform_set_error(
|
|
2704
|
+
error, TF_TRANSFORM_ALLOCATION, "plan JSON allocation failed");
|
|
2705
|
+
failed:
|
|
2706
|
+
cJSON_Delete(input_schema);
|
|
2707
|
+
cJSON_Delete(output_schema);
|
|
2708
|
+
cJSON_Delete(recipe);
|
|
2709
|
+
cJSON_Delete(steps);
|
|
2710
|
+
cJSON_Delete(root);
|
|
2711
|
+
return NULL;
|
|
2712
|
+
}
|
|
2713
|
+
|
|
2714
|
+
static tf_transform_code parse_schema_json(
|
|
2715
|
+
const cJSON *array, int is_output,
|
|
2716
|
+
const tf_transform_runtime_copy *runtime,
|
|
2717
|
+
tf_transform_resource_ledger *ledger,
|
|
2718
|
+
tf_transform_schema *out, tf_transform_error **error) {
|
|
2719
|
+
static const char *const input_keys[] = {"dtype", "id", "name"};
|
|
2720
|
+
static const char *const output_keys[] = {
|
|
2721
|
+
"category", "dtype", "id", "name", "role", "sourceId"
|
|
2722
|
+
};
|
|
2723
|
+
tf_field_view_v1 *fields = NULL;
|
|
2724
|
+
tf_schema_view_v1 view;
|
|
2725
|
+
tf_transform_code code;
|
|
2726
|
+
size_t count = 0;
|
|
2727
|
+
const cJSON *entry;
|
|
2728
|
+
if (!cJSON_IsArray(array))
|
|
2729
|
+
return tf_transform_set_error(
|
|
2730
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema is not a nonempty array");
|
|
2731
|
+
if (!runtime) return tf_transform_set_error(
|
|
2732
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2733
|
+
"plan schema runtime is null");
|
|
2734
|
+
cJSON_ArrayForEach(entry, array) {
|
|
2735
|
+
if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
2736
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
2737
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2738
|
+
}
|
|
2739
|
+
if (count == SIZE_MAX) return tf_transform_set_error(
|
|
2740
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
2741
|
+
"plan schema column count overflows");
|
|
2742
|
+
++count;
|
|
2743
|
+
}
|
|
2744
|
+
if (count == 0) return tf_transform_set_error(
|
|
2745
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2746
|
+
"plan schema is not a nonempty array");
|
|
2747
|
+
if ((uint64_t)count > (is_output ? runtime->limits.max_output_columns
|
|
2748
|
+
: runtime->limits.max_input_columns)
|
|
2749
|
+
|| (size_t)count > SIZE_MAX / sizeof(*fields)
|
|
2750
|
+
|| (uint64_t)count * sizeof(*fields)
|
|
2751
|
+
> runtime->limits.max_allocation_bytes)
|
|
2752
|
+
return tf_transform_set_error(
|
|
2753
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "plan schema exceeds limits");
|
|
2754
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
2755
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2756
|
+
fields = (tf_field_view_v1 *)tf_transform_resource_calloc(
|
|
2757
|
+
ledger, count, sizeof(*fields), error);
|
|
2758
|
+
if (!fields) return ledger->last_code != TF_TRANSFORM_OK
|
|
2759
|
+
? ledger->last_code : tf_transform_set_error(
|
|
2760
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
2761
|
+
"plan schema view allocation failed");
|
|
2762
|
+
entry = array->child;
|
|
2763
|
+
for (size_t i = 0; i < count; ++i, entry = entry->next) {
|
|
2764
|
+
const cJSON *dtype;
|
|
2765
|
+
const cJSON *id;
|
|
2766
|
+
const cJSON *name;
|
|
2767
|
+
size_t id_len;
|
|
2768
|
+
size_t name_len;
|
|
2769
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
2770
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
2771
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2772
|
+
}
|
|
2773
|
+
if (!exact_keys(entry, is_output ? output_keys : input_keys,
|
|
2774
|
+
is_output ? 6 : 3)) {
|
|
2775
|
+
code = tf_transform_set_error(
|
|
2776
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema field shape is invalid");
|
|
2777
|
+
goto done;
|
|
2778
|
+
}
|
|
2779
|
+
dtype = required_item(entry, "dtype");
|
|
2780
|
+
id = required_item(entry, "id");
|
|
2781
|
+
name = required_item(entry, "name");
|
|
2782
|
+
if ((!json_string_equals(dtype, "float32")
|
|
2783
|
+
&& !json_string_equals(dtype, "float64"))
|
|
2784
|
+
|| !cJSON_IsString(id) || !id->valuestring
|
|
2785
|
+
|| !cJSON_IsString(name) || !name->valuestring) {
|
|
2786
|
+
code = tf_transform_set_error(
|
|
2787
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema field is invalid");
|
|
2788
|
+
goto done;
|
|
2789
|
+
}
|
|
2790
|
+
if (is_output) {
|
|
2791
|
+
static const char *const other_keys[] = {"t"};
|
|
2792
|
+
const cJSON *category = required_item(entry, "category");
|
|
2793
|
+
const cJSON *role = required_item(entry, "role");
|
|
2794
|
+
const cJSON *source_id = required_item(entry, "sourceId");
|
|
2795
|
+
uint64_t category_bits = 0;
|
|
2796
|
+
uint32_t category_dtype = 0;
|
|
2797
|
+
size_t source_id_len = 0;
|
|
2798
|
+
if (cJSON_IsString(source_id) && source_id->valuestring) {
|
|
2799
|
+
code = runtime_cstring_length(
|
|
2800
|
+
source_id->valuestring,
|
|
2801
|
+
(size_t)runtime->limits.max_string_bytes,
|
|
2802
|
+
runtime, &source_id_len, error);
|
|
2803
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2804
|
+
}
|
|
2805
|
+
code = runtime_cstring_length(
|
|
2806
|
+
id->valuestring, (size_t)runtime->limits.max_string_bytes,
|
|
2807
|
+
runtime, &id_len, error);
|
|
2808
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2809
|
+
if ((!json_string_equals(role, "value")
|
|
2810
|
+
&& !json_string_equals(role, "label")
|
|
2811
|
+
&& !json_string_equals(role, "onehot"))
|
|
2812
|
+
|| !cJSON_IsString(source_id) || !source_id->valuestring
|
|
2813
|
+
|| !json_string_equals(dtype, "float64")) {
|
|
2814
|
+
code = tf_transform_set_error(
|
|
2815
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2816
|
+
"output schema metadata is invalid");
|
|
2817
|
+
goto done;
|
|
2818
|
+
}
|
|
2819
|
+
if (json_string_equals(role, "onehot")) {
|
|
2820
|
+
if (!parse_tagged_bits(
|
|
2821
|
+
category, &category_bits, &category_dtype)
|
|
2822
|
+
&& !(exact_keys(category, other_keys, 1)
|
|
2823
|
+
&& json_string_equals(
|
|
2824
|
+
required_item(category, "t"), "other"))) {
|
|
2825
|
+
code = tf_transform_set_error(
|
|
2826
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2827
|
+
"one-hot output category is invalid");
|
|
2828
|
+
goto done;
|
|
2829
|
+
}
|
|
2830
|
+
} else if (!cJSON_IsNull(category)) {
|
|
2831
|
+
code = tf_transform_set_error(
|
|
2832
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2833
|
+
"non-one-hot output category is invalid");
|
|
2834
|
+
goto done;
|
|
2835
|
+
}
|
|
2836
|
+
}
|
|
2837
|
+
fields[i].abi_version = 1;
|
|
2838
|
+
fields[i].struct_size = (uint32_t)sizeof(fields[i]);
|
|
2839
|
+
fields[i].dtype = json_string_equals(dtype, "float32")
|
|
2840
|
+
? TF_VIEW_FLOAT32 : TF_VIEW_FLOAT64;
|
|
2841
|
+
if (!is_output) {
|
|
2842
|
+
code = runtime_cstring_length(
|
|
2843
|
+
id->valuestring, (size_t)runtime->limits.max_string_bytes,
|
|
2844
|
+
runtime, &id_len, error);
|
|
2845
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2846
|
+
}
|
|
2847
|
+
code = runtime_cstring_length(
|
|
2848
|
+
name->valuestring, (size_t)runtime->limits.max_string_bytes,
|
|
2849
|
+
runtime, &name_len, error);
|
|
2850
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
2851
|
+
fields[i].id_utf8 = (const uint8_t *)id->valuestring;
|
|
2852
|
+
fields[i].id_bytes = id_len;
|
|
2853
|
+
fields[i].name_utf8 = (const uint8_t *)name->valuestring;
|
|
2854
|
+
fields[i].name_bytes = name_len;
|
|
2855
|
+
}
|
|
2856
|
+
memset(&view, 0, sizeof(view));
|
|
2857
|
+
view.abi_version = 1;
|
|
2858
|
+
view.struct_size = (uint32_t)sizeof(view);
|
|
2859
|
+
view.column_count = count;
|
|
2860
|
+
view.fields = fields;
|
|
2861
|
+
view.fields_bytes = count * sizeof(*fields);
|
|
2862
|
+
code = tf_transform_schema_copy_runtime_ledger(
|
|
2863
|
+
&view, runtime, ledger, out, error);
|
|
2864
|
+
if (code == TF_TRANSFORM_INVALID_ARGUMENT)
|
|
2865
|
+
code = tf_transform_set_error(
|
|
2866
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan schema validation failed");
|
|
2867
|
+
if (code == TF_TRANSFORM_OK) out->is_output = is_output;
|
|
2868
|
+
if (code == TF_TRANSFORM_OK && is_output) {
|
|
2869
|
+
entry = array->child;
|
|
2870
|
+
for (size_t i = 0; i < count; ++i, entry = entry->next) {
|
|
2871
|
+
static const char *const other_keys[] = {"t"};
|
|
2872
|
+
const cJSON *category = required_item(entry, "category");
|
|
2873
|
+
const cJSON *role = required_item(entry, "role");
|
|
2874
|
+
const cJSON *source_id = required_item(entry, "sourceId");
|
|
2875
|
+
size_t source_id_len = 0;
|
|
2876
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
2877
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
2878
|
+
if (code != TF_TRANSFORM_OK) break;
|
|
2879
|
+
}
|
|
2880
|
+
code = runtime_cstring_length(
|
|
2881
|
+
source_id->valuestring,
|
|
2882
|
+
(size_t)runtime->limits.max_string_bytes,
|
|
2883
|
+
runtime, &source_id_len, error);
|
|
2884
|
+
if (code != TF_TRANSFORM_OK) break;
|
|
2885
|
+
out->fields[i].source_id = (char *)tf_transform_resource_malloc(
|
|
2886
|
+
ledger, source_id_len + 1, error);
|
|
2887
|
+
if (!out->fields[i].source_id) {
|
|
2888
|
+
code = ledger->last_code != TF_TRANSFORM_OK
|
|
2889
|
+
? ledger->last_code : tf_transform_set_error(
|
|
2890
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
2891
|
+
"output schema source ID allocation failed");
|
|
2892
|
+
break;
|
|
2893
|
+
}
|
|
2894
|
+
code = tf_transform_copy_bytes_runtime(
|
|
2895
|
+
out->fields[i].source_id, source_id->valuestring,
|
|
2896
|
+
source_id_len + 1, runtime, error);
|
|
2897
|
+
if (code != TF_TRANSFORM_OK) break;
|
|
2898
|
+
out->fields[i].source_id_len = source_id_len;
|
|
2899
|
+
if (json_string_equals(role, "label"))
|
|
2900
|
+
out->fields[i].role = TF_TRANSFORM_ROLE_LABEL;
|
|
2901
|
+
else if (json_string_equals(role, "onehot"))
|
|
2902
|
+
out->fields[i].role = TF_TRANSFORM_ROLE_ONEHOT;
|
|
2903
|
+
else out->fields[i].role = TF_TRANSFORM_ROLE_VALUE;
|
|
2904
|
+
if (out->fields[i].role == TF_TRANSFORM_ROLE_ONEHOT) {
|
|
2905
|
+
uint64_t bits = 0;
|
|
2906
|
+
uint32_t category_dtype = 0;
|
|
2907
|
+
if (parse_tagged_bits(category, &bits, &category_dtype)) {
|
|
2908
|
+
out->fields[i].category_kind
|
|
2909
|
+
= TF_TRANSFORM_SCHEMA_CATEGORY_VALUE;
|
|
2910
|
+
out->fields[i].category_dtype = category_dtype;
|
|
2911
|
+
out->fields[i].category_bits = bits;
|
|
2912
|
+
} else if (exact_keys(category, other_keys, 1)
|
|
2913
|
+
&& json_string_equals(
|
|
2914
|
+
required_item(category, "t"), "other")) {
|
|
2915
|
+
out->fields[i].category_kind
|
|
2916
|
+
= TF_TRANSFORM_SCHEMA_CATEGORY_OTHER;
|
|
2917
|
+
} else {
|
|
2918
|
+
code = tf_transform_set_error(
|
|
2919
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
2920
|
+
"one-hot output category is invalid");
|
|
2921
|
+
break;
|
|
2922
|
+
}
|
|
2923
|
+
} else {
|
|
2924
|
+
out->fields[i].category_kind
|
|
2925
|
+
= TF_TRANSFORM_SCHEMA_CATEGORY_NONE;
|
|
2926
|
+
}
|
|
2927
|
+
}
|
|
2928
|
+
if (code != TF_TRANSFORM_OK) tf_transform_schema_clear(out);
|
|
2929
|
+
}
|
|
2930
|
+
done:
|
|
2931
|
+
free(fields);
|
|
2932
|
+
tf_transform_resource_release(
|
|
2933
|
+
ledger, (uint64_t)count * sizeof(*fields));
|
|
2934
|
+
return code;
|
|
2935
|
+
}
|
|
2936
|
+
|
|
2937
|
+
static tf_transform_code lowercase_sha256_string_runtime(
|
|
2938
|
+
const cJSON *value, const tf_transform_runtime_copy *runtime,
|
|
2939
|
+
int *valid, tf_transform_error **error) {
|
|
2940
|
+
size_t length = 0;
|
|
2941
|
+
tf_transform_code code;
|
|
2942
|
+
if (!valid) return tf_transform_set_error(
|
|
2943
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT,
|
|
2944
|
+
"fingerprint validation output is null");
|
|
2945
|
+
*valid = 0;
|
|
2946
|
+
if (!cJSON_IsString(value) || !value->valuestring) return TF_TRANSFORM_OK;
|
|
2947
|
+
code = runtime_cstring_length(
|
|
2948
|
+
value->valuestring, (size_t)runtime->limits.max_string_bytes,
|
|
2949
|
+
runtime, &length, error);
|
|
2950
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
2951
|
+
if (length != 64) return TF_TRANSFORM_OK;
|
|
2952
|
+
for (size_t i = 0; i < 64; ++i) {
|
|
2953
|
+
unsigned char c = (unsigned char)value->valuestring[i];
|
|
2954
|
+
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')))
|
|
2955
|
+
return TF_TRANSFORM_OK;
|
|
2956
|
+
}
|
|
2957
|
+
*valid = 1;
|
|
2958
|
+
return TF_TRANSFORM_OK;
|
|
2959
|
+
}
|
|
2960
|
+
|
|
2961
|
+
static int tagged_f64(const cJSON *value, double *out) {
|
|
2962
|
+
uint32_t dtype = 0;
|
|
2963
|
+
return parse_tagged_constant(value, out, &dtype) && dtype == TF_VIEW_FLOAT64;
|
|
2964
|
+
}
|
|
2965
|
+
|
|
2966
|
+
static int tagged_category_bits(
|
|
2967
|
+
const cJSON *value, uint32_t expected_dtype, uint64_t *out) {
|
|
2968
|
+
uint64_t decoded = 0;
|
|
2969
|
+
uint32_t dtype = 0;
|
|
2970
|
+
if (!out || !parse_tagged_bits(value, &decoded, &dtype)
|
|
2971
|
+
|| dtype != expected_dtype)
|
|
2972
|
+
return 0;
|
|
2973
|
+
if (dtype == TF_VIEW_FLOAT32) {
|
|
2974
|
+
uint32_t bits = (uint32_t)decoded;
|
|
2975
|
+
if ((bits & UINT32_C(0x7f800000)) == UINT32_C(0x7f800000)
|
|
2976
|
+
|| bits == UINT32_C(0x80000000)) return 0;
|
|
2977
|
+
*out = bits;
|
|
2978
|
+
return 1;
|
|
2979
|
+
}
|
|
2980
|
+
if ((decoded & UINT64_C(0x7ff0000000000000))
|
|
2981
|
+
== UINT64_C(0x7ff0000000000000)
|
|
2982
|
+
|| decoded == UINT64_C(0x8000000000000000)) return 0;
|
|
2983
|
+
*out = decoded;
|
|
2984
|
+
return 1;
|
|
2985
|
+
}
|
|
2986
|
+
|
|
2987
|
+
static int category_bits_are_integer(uint64_t bits, uint32_t dtype) {
|
|
2988
|
+
double integer;
|
|
2989
|
+
double value = tf_transform_category_decode(bits, dtype);
|
|
2990
|
+
return modf(value, &integer) == 0.0;
|
|
2991
|
+
}
|
|
2992
|
+
|
|
2993
|
+
static int null_or_policy(
|
|
2994
|
+
const cJSON *value, tf_transform_all_missing policy,
|
|
2995
|
+
tf_transform_impute_op op) {
|
|
2996
|
+
if (op != TF_TRANSFORM_IMPUTE_MEAN
|
|
2997
|
+
&& op != TF_TRANSFORM_IMPUTE_MEDIAN) return cJSON_IsNull(value);
|
|
2998
|
+
return json_string_equals(value,
|
|
2999
|
+
policy == TF_TRANSFORM_ALL_MISSING_ZERO ? "zero" : "error");
|
|
3000
|
+
}
|
|
3001
|
+
|
|
3002
|
+
static tf_transform_code parse_numeric_step(
|
|
3003
|
+
const cJSON *value, const tf_transform_recipe_column *recipe,
|
|
3004
|
+
const tf_transform_runtime_copy *runtime,
|
|
3005
|
+
tf_transform_numeric_state *out, tf_transform_error **error) {
|
|
3006
|
+
static const char *const step_keys[] = {
|
|
3007
|
+
"categorical", "kind", "numeric", "sourceId", "stateVersion"
|
|
3008
|
+
};
|
|
3009
|
+
static const char *const numeric_keys[] = {"impute", "normalize"};
|
|
3010
|
+
static const char *const impute_keys[] = {"allMissing", "op", "value"};
|
|
3011
|
+
static const char *const normalize_keys[] = {
|
|
3012
|
+
"ddof", "location", "op", "scale"
|
|
3013
|
+
};
|
|
3014
|
+
const cJSON *numeric;
|
|
3015
|
+
const cJSON *impute;
|
|
3016
|
+
const cJSON *normalize;
|
|
3017
|
+
const cJSON *impute_value;
|
|
3018
|
+
const cJSON *ddof;
|
|
3019
|
+
uint64_t state_version;
|
|
3020
|
+
uint64_t parsed_ddof;
|
|
3021
|
+
double location;
|
|
3022
|
+
double scale;
|
|
3023
|
+
double resolved = 0.0;
|
|
3024
|
+
const cJSON *source_id = required_item(value, "sourceId");
|
|
3025
|
+
size_t source_id_len = 0;
|
|
3026
|
+
int source_equal = 0;
|
|
3027
|
+
tf_transform_code code;
|
|
3028
|
+
memset(out, 0, sizeof(*out));
|
|
3029
|
+
if (cJSON_IsString(source_id) && source_id->valuestring) {
|
|
3030
|
+
code = runtime_cstring_length(
|
|
3031
|
+
source_id->valuestring,
|
|
3032
|
+
(size_t)runtime->limits.max_string_bytes,
|
|
3033
|
+
runtime, &source_id_len, error);
|
|
3034
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3035
|
+
code = runtime_span_equal(
|
|
3036
|
+
source_id->valuestring, source_id_len,
|
|
3037
|
+
recipe->source_id, recipe->source_id_len,
|
|
3038
|
+
runtime, &source_equal, error);
|
|
3039
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3040
|
+
}
|
|
3041
|
+
if (!exact_keys(value, step_keys, 5)
|
|
3042
|
+
|| !cJSON_IsNull(required_item(value, "categorical"))
|
|
3043
|
+
|| !json_string_equals(required_item(value, "kind"), "numeric")
|
|
3044
|
+
|| !source_equal
|
|
3045
|
+
|| !safe_uint64(required_item(value, "stateVersion"), &state_version))
|
|
3046
|
+
return tf_transform_set_error(
|
|
3047
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "numeric plan step shape is invalid");
|
|
3048
|
+
if (state_version != 1)
|
|
3049
|
+
return tf_transform_set_error(
|
|
3050
|
+
error, TF_TRANSFORM_UNSUPPORTED_VERSION,
|
|
3051
|
+
"numeric plan state version is unsupported");
|
|
3052
|
+
numeric = required_item(value, "numeric");
|
|
3053
|
+
if (!exact_keys(numeric, numeric_keys, 2)) goto corrupt;
|
|
3054
|
+
impute = required_item(numeric, "impute");
|
|
3055
|
+
normalize = required_item(numeric, "normalize");
|
|
3056
|
+
if (!exact_keys(impute, impute_keys, 3)
|
|
3057
|
+
|| !json_string_equals(required_item(impute, "op"),
|
|
3058
|
+
impute_op_name(recipe->impute))
|
|
3059
|
+
|| !null_or_policy(required_item(impute, "allMissing"),
|
|
3060
|
+
recipe->all_missing, recipe->impute)) goto corrupt;
|
|
3061
|
+
impute_value = required_item(impute, "value");
|
|
3062
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_NONE) {
|
|
3063
|
+
if (!cJSON_IsNull(impute_value)) goto corrupt;
|
|
3064
|
+
} else {
|
|
3065
|
+
if (!tagged_f64(impute_value, &resolved)) goto corrupt;
|
|
3066
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_ZERO
|
|
3067
|
+
&& tf_transform_double_bits(resolved) != 0) goto corrupt;
|
|
3068
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_MEDIAN
|
|
3069
|
+
&& resolved == 0.0
|
|
3070
|
+
&& tf_transform_double_bits(resolved) != 0) goto corrupt;
|
|
3071
|
+
if (recipe->impute == TF_TRANSFORM_IMPUTE_CONSTANT
|
|
3072
|
+
&& tf_transform_double_bits(resolved)
|
|
3073
|
+
!= tf_transform_double_bits(recipe->constant)) goto corrupt;
|
|
3074
|
+
out->has_impute_value = 1;
|
|
3075
|
+
out->impute_value = resolved;
|
|
3076
|
+
}
|
|
3077
|
+
if (!exact_keys(normalize, normalize_keys, 4)
|
|
3078
|
+
|| !json_string_equals(required_item(normalize, "op"),
|
|
3079
|
+
normalize_op_name(recipe->normalize))
|
|
3080
|
+
|| !tagged_f64(required_item(normalize, "location"), &location)
|
|
3081
|
+
|| !tagged_f64(required_item(normalize, "scale"), &scale)
|
|
3082
|
+
|| scale <= 0.0) goto corrupt;
|
|
3083
|
+
ddof = required_item(normalize, "ddof");
|
|
3084
|
+
if (recipe->normalize == TF_TRANSFORM_NORMALIZE_STANDARD) {
|
|
3085
|
+
if (!safe_uint64(ddof, &parsed_ddof)
|
|
3086
|
+
|| parsed_ddof != (uint64_t)recipe->ddof) goto corrupt;
|
|
3087
|
+
} else if (!cJSON_IsNull(ddof)) goto corrupt;
|
|
3088
|
+
if (recipe->normalize == TF_TRANSFORM_NORMALIZE_NONE
|
|
3089
|
+
&& (tf_transform_double_bits(location) != 0
|
|
3090
|
+
|| tf_transform_double_bits(scale) != UINT64_C(0x3ff0000000000000)))
|
|
3091
|
+
goto corrupt;
|
|
3092
|
+
out->impute = recipe->impute;
|
|
3093
|
+
out->all_missing = recipe->all_missing;
|
|
3094
|
+
out->normalize = recipe->normalize;
|
|
3095
|
+
out->ddof = recipe->ddof;
|
|
3096
|
+
out->location = location;
|
|
3097
|
+
out->scale = scale;
|
|
3098
|
+
return TF_TRANSFORM_OK;
|
|
3099
|
+
corrupt:
|
|
3100
|
+
return tf_transform_set_error(
|
|
3101
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "numeric plan state is inconsistent");
|
|
3102
|
+
}
|
|
3103
|
+
|
|
3104
|
+
static tf_transform_code parse_categorical_step(
|
|
3105
|
+
const cJSON *value, const tf_transform_recipe_column *recipe,
|
|
3106
|
+
uint32_t source_dtype, const tf_transform_runtime_copy *runtime,
|
|
3107
|
+
tf_transform_resource_ledger *ledger, uint64_t *total_categories,
|
|
3108
|
+
tf_transform_categorical_state *out, tf_transform_error **error) {
|
|
3109
|
+
static const char *const step_keys[] = {
|
|
3110
|
+
"categorical", "kind", "numeric", "sourceId", "stateVersion"
|
|
3111
|
+
};
|
|
3112
|
+
static const char *const categorical_keys[] = {
|
|
3113
|
+
"categories", "encode", "impute"
|
|
3114
|
+
};
|
|
3115
|
+
static const char *const encode_keys[] = {
|
|
3116
|
+
"op", "otherOrdinal", "sentinelLabel", "unknown"
|
|
3117
|
+
};
|
|
3118
|
+
static const char *const impute_keys[] = {"allMissing", "op", "value"};
|
|
3119
|
+
const cJSON *categorical;
|
|
3120
|
+
const cJSON *categories;
|
|
3121
|
+
const cJSON *encode;
|
|
3122
|
+
const cJSON *impute;
|
|
3123
|
+
const cJSON *entry;
|
|
3124
|
+
const cJSON *source_id = required_item(value, "sourceId");
|
|
3125
|
+
size_t source_id_len = 0;
|
|
3126
|
+
size_t count = 0;
|
|
3127
|
+
size_t bytes = 0;
|
|
3128
|
+
uint64_t state_version;
|
|
3129
|
+
uint64_t mode_bits;
|
|
3130
|
+
uint64_t other_ordinal = 0;
|
|
3131
|
+
uint64_t previous_category_bits = 0;
|
|
3132
|
+
int64_t sentinel_label = 0;
|
|
3133
|
+
int empty_categories_allowed = 0;
|
|
3134
|
+
int source_equal = 0;
|
|
3135
|
+
int mode_known = 0;
|
|
3136
|
+
tf_transform_code code;
|
|
3137
|
+
memset(out, 0, sizeof(*out));
|
|
3138
|
+
if (cJSON_IsString(source_id) && source_id->valuestring) {
|
|
3139
|
+
code = runtime_cstring_length(
|
|
3140
|
+
source_id->valuestring, (size_t)runtime->limits.max_string_bytes,
|
|
3141
|
+
runtime, &source_id_len, error);
|
|
3142
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3143
|
+
code = runtime_span_equal(
|
|
3144
|
+
source_id->valuestring, source_id_len,
|
|
3145
|
+
recipe->source_id, recipe->source_id_len,
|
|
3146
|
+
runtime, &source_equal, error);
|
|
3147
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3148
|
+
}
|
|
3149
|
+
if (!exact_keys(value, step_keys, 5)
|
|
3150
|
+
|| !json_string_equals(required_item(value, "kind"), "categorical")
|
|
3151
|
+
|| !cJSON_IsNull(required_item(value, "numeric"))
|
|
3152
|
+
|| !source_equal
|
|
3153
|
+
|| !safe_uint64(required_item(value, "stateVersion"), &state_version))
|
|
3154
|
+
goto corrupt;
|
|
3155
|
+
if (state_version != 1)
|
|
3156
|
+
return tf_transform_set_error(
|
|
3157
|
+
error, TF_TRANSFORM_UNSUPPORTED_VERSION,
|
|
3158
|
+
"categorical plan state version is unsupported");
|
|
3159
|
+
categorical = required_item(value, "categorical");
|
|
3160
|
+
if (!exact_keys(categorical, categorical_keys, 3)) goto corrupt;
|
|
3161
|
+
categories = required_item(categorical, "categories");
|
|
3162
|
+
encode = required_item(categorical, "encode");
|
|
3163
|
+
impute = required_item(categorical, "impute");
|
|
3164
|
+
if (!cJSON_IsArray(categories)
|
|
3165
|
+
|| !exact_keys(encode, encode_keys, 4)
|
|
3166
|
+
|| !exact_keys(impute, impute_keys, 3))
|
|
3167
|
+
goto corrupt;
|
|
3168
|
+
if (recipe->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE) {
|
|
3169
|
+
if (!json_string_equals(required_item(impute, "op"), "none")
|
|
3170
|
+
|| !cJSON_IsNull(required_item(impute, "allMissing"))
|
|
3171
|
+
|| !cJSON_IsNull(required_item(impute, "value"))) goto corrupt;
|
|
3172
|
+
} else if (recipe->categorical_impute
|
|
3173
|
+
== TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE) {
|
|
3174
|
+
if (!json_string_equals(required_item(impute, "op"), "mode")
|
|
3175
|
+
|| !json_string_equals(
|
|
3176
|
+
required_item(impute, "allMissing"),
|
|
3177
|
+
recipe->categorical_all_missing == TF_TRANSFORM_ALL_MISSING_ZERO
|
|
3178
|
+
? "zero" : "error")) goto corrupt;
|
|
3179
|
+
} else goto corrupt;
|
|
3180
|
+
if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_NONE) {
|
|
3181
|
+
if (!json_string_equals(required_item(encode, "op"), "none")
|
|
3182
|
+
|| !cJSON_IsNull(required_item(encode, "otherOrdinal"))
|
|
3183
|
+
|| !cJSON_IsNull(required_item(encode, "sentinelLabel"))
|
|
3184
|
+
|| !cJSON_IsNull(required_item(encode, "unknown"))) goto corrupt;
|
|
3185
|
+
} else if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_LABEL) {
|
|
3186
|
+
if (!json_string_equals(required_item(encode, "op"), "label")) goto corrupt;
|
|
3187
|
+
if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_ERROR) {
|
|
3188
|
+
if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
|
|
3189
|
+
|| !cJSON_IsNull(required_item(encode, "sentinelLabel"))
|
|
3190
|
+
|| !json_string_equals(required_item(encode, "unknown"), "error"))
|
|
3191
|
+
goto corrupt;
|
|
3192
|
+
} else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_SENTINEL) {
|
|
3193
|
+
if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
|
|
3194
|
+
|| !safe_int64(
|
|
3195
|
+
required_item(encode, "sentinelLabel"), &sentinel_label)
|
|
3196
|
+
|| sentinel_label != recipe->categorical_sentinel_label
|
|
3197
|
+
|| !json_string_equals(
|
|
3198
|
+
required_item(encode, "unknown"), "sentinel")) goto corrupt;
|
|
3199
|
+
} else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
|
|
3200
|
+
if (!safe_uint64(
|
|
3201
|
+
required_item(encode, "otherOrdinal"), &other_ordinal)
|
|
3202
|
+
|| !cJSON_IsNull(required_item(encode, "sentinelLabel"))
|
|
3203
|
+
|| !json_string_equals(required_item(encode, "unknown"), "other"))
|
|
3204
|
+
goto corrupt;
|
|
3205
|
+
} else goto corrupt;
|
|
3206
|
+
} else if (recipe->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT) {
|
|
3207
|
+
if (!json_string_equals(required_item(encode, "op"), "onehot")
|
|
3208
|
+
|| !cJSON_IsNull(required_item(encode, "sentinelLabel")))
|
|
3209
|
+
goto corrupt;
|
|
3210
|
+
if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_ERROR) {
|
|
3211
|
+
if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
|
|
3212
|
+
|| !json_string_equals(
|
|
3213
|
+
required_item(encode, "unknown"), "error")) goto corrupt;
|
|
3214
|
+
} else if (recipe->categorical_unknown
|
|
3215
|
+
== TF_TRANSFORM_UNKNOWN_ALL_ZERO) {
|
|
3216
|
+
if (!cJSON_IsNull(required_item(encode, "otherOrdinal"))
|
|
3217
|
+
|| !json_string_equals(
|
|
3218
|
+
required_item(encode, "unknown"), "all_zero"))
|
|
3219
|
+
goto corrupt;
|
|
3220
|
+
} else if (recipe->categorical_unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
|
|
3221
|
+
if (!safe_uint64(
|
|
3222
|
+
required_item(encode, "otherOrdinal"), &other_ordinal)
|
|
3223
|
+
|| !json_string_equals(
|
|
3224
|
+
required_item(encode, "unknown"), "other")) goto corrupt;
|
|
3225
|
+
} else goto corrupt;
|
|
3226
|
+
} else goto corrupt;
|
|
3227
|
+
cJSON_ArrayForEach(entry, categories) {
|
|
3228
|
+
uint64_t bits;
|
|
3229
|
+
if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3230
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3231
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3232
|
+
}
|
|
3233
|
+
if (!tagged_category_bits(entry, source_dtype, &bits)
|
|
3234
|
+
|| (recipe->kind == TF_TRANSFORM_KIND_INFER
|
|
3235
|
+
&& !category_bits_are_integer(bits, source_dtype))
|
|
3236
|
+
|| (count != 0 && tf_transform_category_compare(
|
|
3237
|
+
previous_category_bits, bits, source_dtype) >= 0))
|
|
3238
|
+
goto corrupt;
|
|
3239
|
+
if (recipe->categorical_fixed_count
|
|
3240
|
+
&& (count >= recipe->categorical_fixed_count
|
|
3241
|
+
|| bits != recipe->categorical_fixed[count].bits)) goto corrupt;
|
|
3242
|
+
previous_category_bits = bits;
|
|
3243
|
+
if (count == SIZE_MAX) return tf_transform_set_error(
|
|
3244
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3245
|
+
"categorical plan category count overflows");
|
|
3246
|
+
++count;
|
|
3247
|
+
}
|
|
3248
|
+
empty_categories_allowed
|
|
3249
|
+
= recipe->kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
3250
|
+
&& recipe->categorical_impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
3251
|
+
&& recipe->categorical_encode == TF_TRANSFORM_ENCODE_NONE;
|
|
3252
|
+
if (count == 0 && !empty_categories_allowed) goto corrupt;
|
|
3253
|
+
if (count != 0 && empty_categories_allowed) goto corrupt;
|
|
3254
|
+
if (recipe->kind == TF_TRANSFORM_KIND_INFER
|
|
3255
|
+
&& (count < 2 || (uint64_t)count > recipe->infer_max_categories))
|
|
3256
|
+
goto corrupt;
|
|
3257
|
+
if (recipe->categorical_fixed_count && count != recipe->categorical_fixed_count)
|
|
3258
|
+
goto corrupt;
|
|
3259
|
+
if ((uint64_t)count > runtime->limits.max_categories_per_column
|
|
3260
|
+
|| *total_categories > UINT64_MAX - (uint64_t)count
|
|
3261
|
+
|| *total_categories + (uint64_t)count
|
|
3262
|
+
> runtime->limits.max_total_categories
|
|
3263
|
+
|| count > SIZE_MAX / sizeof(*out->categories))
|
|
3264
|
+
return tf_transform_set_error(
|
|
3265
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3266
|
+
"categorical plan state exceeds category limits");
|
|
3267
|
+
if ((recipe->categorical_encode == TF_TRANSFORM_ENCODE_LABEL
|
|
3268
|
+
|| recipe->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT)
|
|
3269
|
+
&& (uint64_t)count > (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1)
|
|
3270
|
+
return tf_transform_set_error(
|
|
3271
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3272
|
+
"categorical encoding count exceeds the safe-integer domain");
|
|
3273
|
+
bytes = count * sizeof(*out->categories);
|
|
3274
|
+
if ((uint64_t)bytes > runtime->limits.max_allocation_bytes)
|
|
3275
|
+
return tf_transform_set_error(
|
|
3276
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3277
|
+
"categorical plan allocation exceeds limits");
|
|
3278
|
+
if (count != 0) {
|
|
3279
|
+
out->categories = (tf_transform_category_value *)
|
|
3280
|
+
tf_transform_resource_calloc(
|
|
3281
|
+
ledger, count, sizeof(*out->categories), error);
|
|
3282
|
+
if (!out->categories)
|
|
3283
|
+
return ledger->last_code != TF_TRANSFORM_OK
|
|
3284
|
+
? ledger->last_code : tf_transform_set_error(
|
|
3285
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
3286
|
+
"categorical plan state allocation failed");
|
|
3287
|
+
}
|
|
3288
|
+
out->source_dtype = source_dtype;
|
|
3289
|
+
out->category_count = count;
|
|
3290
|
+
entry = categories->child;
|
|
3291
|
+
for (size_t i = 0; i < count; ++i, entry = entry->next) {
|
|
3292
|
+
uint64_t bits;
|
|
3293
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3294
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3295
|
+
if (code != TF_TRANSFORM_OK) goto failed;
|
|
3296
|
+
}
|
|
3297
|
+
if (!tagged_category_bits(entry, source_dtype, &bits)
|
|
3298
|
+
|| (recipe->kind == TF_TRANSFORM_KIND_INFER
|
|
3299
|
+
&& !category_bits_are_integer(bits, source_dtype))
|
|
3300
|
+
|| (i != 0 && tf_transform_category_compare(
|
|
3301
|
+
out->categories[i - 1].bits, bits, source_dtype) >= 0))
|
|
3302
|
+
goto corrupt_allocated;
|
|
3303
|
+
out->categories[i].bits = bits;
|
|
3304
|
+
}
|
|
3305
|
+
out->impute = recipe->categorical_impute;
|
|
3306
|
+
out->all_missing = recipe->categorical_all_missing;
|
|
3307
|
+
out->encode = recipe->categorical_encode;
|
|
3308
|
+
out->unknown = recipe->categorical_unknown;
|
|
3309
|
+
if (out->unknown == TF_TRANSFORM_UNKNOWN_SENTINEL) {
|
|
3310
|
+
if (sentinel_label >= 0 && (uint64_t)sentinel_label < (uint64_t)count)
|
|
3311
|
+
goto corrupt_allocated;
|
|
3312
|
+
out->sentinel_label = sentinel_label;
|
|
3313
|
+
out->has_sentinel_label = 1;
|
|
3314
|
+
} else if (out->unknown == TF_TRANSFORM_UNKNOWN_OTHER) {
|
|
3315
|
+
if (other_ordinal != (uint64_t)count) goto corrupt_allocated;
|
|
3316
|
+
out->other_ordinal = other_ordinal;
|
|
3317
|
+
out->has_other_ordinal = 1;
|
|
3318
|
+
}
|
|
3319
|
+
if (out->impute == TF_TRANSFORM_CATEGORICAL_IMPUTE_MODE) {
|
|
3320
|
+
if (!tagged_category_bits(
|
|
3321
|
+
required_item(impute, "value"), source_dtype, &mode_bits))
|
|
3322
|
+
goto corrupt_allocated;
|
|
3323
|
+
out->impute_bits = mode_bits;
|
|
3324
|
+
out->has_impute_value = 1;
|
|
3325
|
+
{
|
|
3326
|
+
size_t mode_ordinal = 0;
|
|
3327
|
+
code = tf_transform_category_lookup(
|
|
3328
|
+
out, mode_bits, runtime, &mode_ordinal, &mode_known, error);
|
|
3329
|
+
}
|
|
3330
|
+
if (code != TF_TRANSFORM_OK) goto failed;
|
|
3331
|
+
if (!mode_known) goto corrupt_allocated;
|
|
3332
|
+
}
|
|
3333
|
+
*total_categories += (uint64_t)count;
|
|
3334
|
+
return TF_TRANSFORM_OK;
|
|
3335
|
+
corrupt_allocated:
|
|
3336
|
+
code = tf_transform_set_error(
|
|
3337
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3338
|
+
"categorical plan state is inconsistent");
|
|
3339
|
+
failed:
|
|
3340
|
+
tf_transform_categorical_state_clear(out);
|
|
3341
|
+
tf_transform_resource_release(ledger, (uint64_t)bytes);
|
|
3342
|
+
return code;
|
|
3343
|
+
corrupt:
|
|
3344
|
+
return tf_transform_set_error(
|
|
3345
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3346
|
+
"categorical plan state is inconsistent");
|
|
3347
|
+
}
|
|
3348
|
+
|
|
3349
|
+
static tf_transform_code imported_step_kind(
|
|
3350
|
+
const cJSON *step, const tf_transform_recipe_column *recipe,
|
|
3351
|
+
tf_transform_column_kind *out, tf_transform_error **error) {
|
|
3352
|
+
const cJSON *kind;
|
|
3353
|
+
tf_transform_column_kind resolved;
|
|
3354
|
+
if (!step || !recipe || !out)
|
|
3355
|
+
return tf_transform_set_error(
|
|
3356
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
3357
|
+
"imported step-kind arguments are invalid");
|
|
3358
|
+
kind = required_item(step, "kind");
|
|
3359
|
+
if (json_string_equals(kind, "numeric"))
|
|
3360
|
+
resolved = TF_TRANSFORM_KIND_NUMERIC;
|
|
3361
|
+
else if (json_string_equals(kind, "categorical"))
|
|
3362
|
+
resolved = TF_TRANSFORM_KIND_CATEGORICAL;
|
|
3363
|
+
else return tf_transform_set_error(
|
|
3364
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3365
|
+
"plan step kind is invalid");
|
|
3366
|
+
if ((recipe->kind == TF_TRANSFORM_KIND_NUMERIC
|
|
3367
|
+
&& resolved != TF_TRANSFORM_KIND_NUMERIC)
|
|
3368
|
+
|| (recipe->kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
3369
|
+
&& resolved != TF_TRANSFORM_KIND_CATEGORICAL)
|
|
3370
|
+
|| (recipe->kind != TF_TRANSFORM_KIND_NUMERIC
|
|
3371
|
+
&& recipe->kind != TF_TRANSFORM_KIND_CATEGORICAL
|
|
3372
|
+
&& recipe->kind != TF_TRANSFORM_KIND_INFER))
|
|
3373
|
+
return tf_transform_set_error(
|
|
3374
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3375
|
+
"plan step kind disagrees with its recipe");
|
|
3376
|
+
*out = resolved;
|
|
3377
|
+
return TF_TRANSFORM_OK;
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
tf_transform_code tf_transform_plan_from_json(
|
|
3381
|
+
const uint8_t *json, size_t json_len,
|
|
3382
|
+
const tf_transform_runtime_copy *runtime,
|
|
3383
|
+
tf_transform_plan **out, uint8_t **canonical_out,
|
|
3384
|
+
size_t *canonical_len_out, tf_transform_error **error) {
|
|
3385
|
+
static const char *const top_keys[] = {
|
|
3386
|
+
"format", "inputSchema", "outputSchema", "recipe",
|
|
3387
|
+
"recipeSha256", "steps", "version"
|
|
3388
|
+
};
|
|
3389
|
+
cJSON *root = NULL;
|
|
3390
|
+
tf_transform_recipe *recipe = NULL;
|
|
3391
|
+
tf_transform_plan *plan = NULL;
|
|
3392
|
+
tf_transform_resource_ledger ledger;
|
|
3393
|
+
tf_transform_runtime_copy parse_runtime;
|
|
3394
|
+
tf_transform_runtime_copy recipe_runtime;
|
|
3395
|
+
tf_transform_code code;
|
|
3396
|
+
uint64_t version;
|
|
3397
|
+
char expected_fingerprint[65];
|
|
3398
|
+
const cJSON *steps;
|
|
3399
|
+
size_t step_count = 0;
|
|
3400
|
+
size_t output_column_count = 0;
|
|
3401
|
+
uint64_t total_categories = 0;
|
|
3402
|
+
uint64_t ast_resident_bytes = 0;
|
|
3403
|
+
if (out) *out = NULL;
|
|
3404
|
+
if (canonical_out) *canonical_out = NULL;
|
|
3405
|
+
if (canonical_len_out) *canonical_len_out = 0;
|
|
3406
|
+
if (!json || json_len == 0 || !runtime || !out
|
|
3407
|
+
|| !canonical_out || !canonical_len_out)
|
|
3408
|
+
return tf_transform_set_error(
|
|
3409
|
+
error, TF_TRANSFORM_INVALID_ARGUMENT, "plan JSON argument is null");
|
|
3410
|
+
parse_runtime = *runtime;
|
|
3411
|
+
parse_runtime.limits.max_categories_per_column
|
|
3412
|
+
= (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
|
|
3413
|
+
parse_runtime.limits.max_total_categories
|
|
3414
|
+
= (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
|
|
3415
|
+
tf_transform_resource_ledger_init(&ledger, &parse_runtime);
|
|
3416
|
+
code = tf_transform_json_parse_bounded_runtime_ledger(
|
|
3417
|
+
json, json_len, &parse_runtime, &ledger, &root,
|
|
3418
|
+
&ast_resident_bytes, NULL, error,
|
|
3419
|
+
TF_TRANSFORM_CORRUPT_PLAN);
|
|
3420
|
+
if (code != TF_TRANSFORM_OK) return code;
|
|
3421
|
+
ledger.runtime = runtime;
|
|
3422
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3423
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3424
|
+
if (!exact_keys(root, top_keys, 7)
|
|
3425
|
+
|| !json_string_equals(required_item(root, "format"),
|
|
3426
|
+
"tranfi.transform-plan")
|
|
3427
|
+
|| !safe_uint64(required_item(root, "version"), &version)) {
|
|
3428
|
+
code = tf_transform_set_error(
|
|
3429
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan JSON header is invalid");
|
|
3430
|
+
goto done;
|
|
3431
|
+
}
|
|
3432
|
+
if (version != 1) {
|
|
3433
|
+
code = tf_transform_set_error(
|
|
3434
|
+
error, TF_TRANSFORM_UNSUPPORTED_VERSION, "plan JSON version is unsupported");
|
|
3435
|
+
goto done;
|
|
3436
|
+
}
|
|
3437
|
+
{
|
|
3438
|
+
const cJSON *embedded_recipe = required_item(root, "recipe");
|
|
3439
|
+
uint64_t recipe_version;
|
|
3440
|
+
uint64_t policy_version;
|
|
3441
|
+
if (cJSON_IsObject(embedded_recipe)
|
|
3442
|
+
&& safe_uint64(required_item(embedded_recipe, "version"),
|
|
3443
|
+
&recipe_version)
|
|
3444
|
+
&& safe_uint64(required_item(embedded_recipe, "policyVersion"),
|
|
3445
|
+
&policy_version)
|
|
3446
|
+
&& (recipe_version != 1 || policy_version != 1)) {
|
|
3447
|
+
code = tf_transform_set_error(
|
|
3448
|
+
error, TF_TRANSFORM_UNSUPPORTED_VERSION,
|
|
3449
|
+
"embedded recipe version is unsupported");
|
|
3450
|
+
goto done;
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
recipe_runtime = *runtime;
|
|
3454
|
+
recipe_runtime.limits.max_output_columns
|
|
3455
|
+
= (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
|
|
3456
|
+
recipe_runtime.limits.max_categories_per_column
|
|
3457
|
+
= (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
|
|
3458
|
+
recipe_runtime.limits.max_total_categories
|
|
3459
|
+
= (uint64_t)TF_TRANSFORM_MAX_SAFE_INTEGER_V1;
|
|
3460
|
+
code = recipe_from_json_value(
|
|
3461
|
+
required_item(root, "recipe"), &recipe_runtime,
|
|
3462
|
+
&ledger, &recipe, error);
|
|
3463
|
+
if (code != TF_TRANSFORM_OK) {
|
|
3464
|
+
if (code == TF_TRANSFORM_INVALID_RECIPE
|
|
3465
|
+
|| code == TF_TRANSFORM_INVALID_ARGUMENT)
|
|
3466
|
+
code = tf_transform_set_error(
|
|
3467
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "embedded recipe is invalid");
|
|
3468
|
+
goto done;
|
|
3469
|
+
}
|
|
3470
|
+
if (!recipe) {
|
|
3471
|
+
code = tf_transform_set_error(
|
|
3472
|
+
error, TF_TRANSFORM_INTERNAL,
|
|
3473
|
+
"embedded recipe parser returned no recipe");
|
|
3474
|
+
goto done;
|
|
3475
|
+
}
|
|
3476
|
+
plan = (tf_transform_plan *)tf_transform_resource_calloc(
|
|
3477
|
+
&ledger, 1, sizeof(*plan), error);
|
|
3478
|
+
if (!plan) {
|
|
3479
|
+
code = ledger.last_code != TF_TRANSFORM_OK
|
|
3480
|
+
? ledger.last_code : tf_transform_set_error(
|
|
3481
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
3482
|
+
"imported plan allocation failed");
|
|
3483
|
+
goto done;
|
|
3484
|
+
}
|
|
3485
|
+
atomic_init(&plan->refcount, 1u);
|
|
3486
|
+
plan->recipe = recipe;
|
|
3487
|
+
recipe = NULL;
|
|
3488
|
+
if (!plan->recipe) {
|
|
3489
|
+
code = tf_transform_set_error(
|
|
3490
|
+
error, TF_TRANSFORM_INTERNAL, "recipe parser returned no recipe");
|
|
3491
|
+
goto done;
|
|
3492
|
+
}
|
|
3493
|
+
code = parse_schema_json(
|
|
3494
|
+
required_item(root, "inputSchema"), 0, runtime,
|
|
3495
|
+
&ledger, &plan->input_schema, error);
|
|
3496
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3497
|
+
{
|
|
3498
|
+
const cJSON *output_schema = required_item(root, "outputSchema");
|
|
3499
|
+
const cJSON *field;
|
|
3500
|
+
if (!cJSON_IsArray(output_schema)) {
|
|
3501
|
+
code = tf_transform_set_error(
|
|
3502
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3503
|
+
"plan output schema is not an array");
|
|
3504
|
+
goto done;
|
|
3505
|
+
}
|
|
3506
|
+
cJSON_ArrayForEach(field, output_schema) {
|
|
3507
|
+
if (output_column_count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3508
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3509
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3510
|
+
}
|
|
3511
|
+
if (output_column_count == SIZE_MAX) {
|
|
3512
|
+
code = tf_transform_set_error(
|
|
3513
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3514
|
+
"plan output schema column count overflows");
|
|
3515
|
+
goto done;
|
|
3516
|
+
}
|
|
3517
|
+
++output_column_count;
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
if (plan->input_schema.field_count != plan->recipe->column_count
|
|
3521
|
+
|| output_column_count == 0
|
|
3522
|
+
|| (uint64_t)output_column_count
|
|
3523
|
+
> plan->recipe->max_output_columns) {
|
|
3524
|
+
code = tf_transform_set_error(
|
|
3525
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3526
|
+
"plan recipe and schema column counts disagree");
|
|
3527
|
+
goto done;
|
|
3528
|
+
}
|
|
3529
|
+
for (size_t i = 0; i < plan->input_schema.field_count; ++i) {
|
|
3530
|
+
const tf_transform_recipe_column *recipe_column = &plan->recipe->columns[i];
|
|
3531
|
+
const tf_transform_schema_field_owned *input = &plan->input_schema.fields[i];
|
|
3532
|
+
int source_equal = 0;
|
|
3533
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3534
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3535
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3536
|
+
}
|
|
3537
|
+
code = runtime_span_equal(
|
|
3538
|
+
recipe_column->source_id, recipe_column->source_id_len,
|
|
3539
|
+
input->id, input->id_len, runtime, &source_equal, error);
|
|
3540
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3541
|
+
if (!source_equal
|
|
3542
|
+
|| (recipe_column->categorical_fixed_count
|
|
3543
|
+
&& recipe_column->categorical_fixed_dtype != input->dtype)
|
|
3544
|
+
|| (recipe_column->kind != TF_TRANSFORM_KIND_CATEGORICAL
|
|
3545
|
+
&& recipe_column->impute == TF_TRANSFORM_IMPUTE_CONSTANT
|
|
3546
|
+
&& recipe_column->constant_dtype != input->dtype)) {
|
|
3547
|
+
code = tf_transform_set_error(
|
|
3548
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3549
|
+
"plan recipe and schema fields disagree");
|
|
3550
|
+
goto done;
|
|
3551
|
+
}
|
|
3552
|
+
}
|
|
3553
|
+
{
|
|
3554
|
+
int valid_fingerprint = 0;
|
|
3555
|
+
code = lowercase_sha256_string_runtime(
|
|
3556
|
+
required_item(root, "recipeSha256"), runtime,
|
|
3557
|
+
&valid_fingerprint, error);
|
|
3558
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3559
|
+
if (!valid_fingerprint) {
|
|
3560
|
+
code = tf_transform_set_error(
|
|
3561
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan recipe fingerprint is invalid");
|
|
3562
|
+
goto done;
|
|
3563
|
+
}
|
|
3564
|
+
}
|
|
3565
|
+
code = imported_recipe_fingerprint(
|
|
3566
|
+
root, runtime, &ledger, expected_fingerprint, error);
|
|
3567
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3568
|
+
if (strcmp(required_item(root, "recipeSha256")->valuestring,
|
|
3569
|
+
expected_fingerprint) != 0) {
|
|
3570
|
+
code = tf_transform_set_error(
|
|
3571
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan recipe fingerprint mismatches");
|
|
3572
|
+
goto done;
|
|
3573
|
+
}
|
|
3574
|
+
steps = required_item(root, "steps");
|
|
3575
|
+
if (!cJSON_IsArray(steps)) {
|
|
3576
|
+
code = tf_transform_set_error(
|
|
3577
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan step count is invalid");
|
|
3578
|
+
goto done;
|
|
3579
|
+
}
|
|
3580
|
+
{
|
|
3581
|
+
const cJSON *step;
|
|
3582
|
+
cJSON_ArrayForEach(step, steps) {
|
|
3583
|
+
if (step_count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3584
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3585
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3586
|
+
}
|
|
3587
|
+
if (step_count == SIZE_MAX) {
|
|
3588
|
+
code = tf_transform_set_error(
|
|
3589
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3590
|
+
"plan step count overflows");
|
|
3591
|
+
goto done;
|
|
3592
|
+
}
|
|
3593
|
+
++step_count;
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
if (step_count != plan->input_schema.field_count) {
|
|
3597
|
+
code = tf_transform_set_error(
|
|
3598
|
+
error, TF_TRANSFORM_CORRUPT_PLAN, "plan step count is invalid");
|
|
3599
|
+
goto done;
|
|
3600
|
+
}
|
|
3601
|
+
{
|
|
3602
|
+
const cJSON *step = steps->child;
|
|
3603
|
+
uint64_t expected_output_columns = 0;
|
|
3604
|
+
uint64_t preflight_total_categories = 0;
|
|
3605
|
+
for (size_t i = 0; i < plan->input_schema.field_count;
|
|
3606
|
+
++i, step = step->next) {
|
|
3607
|
+
const tf_transform_recipe_column *column
|
|
3608
|
+
= &plan->recipe->columns[i];
|
|
3609
|
+
tf_transform_column_kind resolved_kind;
|
|
3610
|
+
uint64_t width = 1;
|
|
3611
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3612
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3613
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3614
|
+
}
|
|
3615
|
+
code = imported_step_kind(
|
|
3616
|
+
step, column, &resolved_kind, error);
|
|
3617
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3618
|
+
if (resolved_kind == TF_TRANSFORM_KIND_CATEGORICAL) {
|
|
3619
|
+
const cJSON *categorical = required_item(step, "categorical");
|
|
3620
|
+
const cJSON *categories = required_item(
|
|
3621
|
+
categorical, "categories");
|
|
3622
|
+
const cJSON *category;
|
|
3623
|
+
size_t count = 0;
|
|
3624
|
+
uint64_t previous_bits = 0;
|
|
3625
|
+
if (!cJSON_IsArray(categories)) {
|
|
3626
|
+
code = tf_transform_set_error(
|
|
3627
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3628
|
+
"categorical plan categories are invalid");
|
|
3629
|
+
goto done;
|
|
3630
|
+
}
|
|
3631
|
+
cJSON_ArrayForEach(category, categories) {
|
|
3632
|
+
uint64_t bits;
|
|
3633
|
+
if (count % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3634
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3635
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3636
|
+
}
|
|
3637
|
+
if (!tagged_category_bits(
|
|
3638
|
+
category, plan->input_schema.fields[i].dtype, &bits)
|
|
3639
|
+
|| (column->kind == TF_TRANSFORM_KIND_INFER
|
|
3640
|
+
&& !category_bits_are_integer(
|
|
3641
|
+
bits, plan->input_schema.fields[i].dtype))
|
|
3642
|
+
|| (count != 0 && tf_transform_category_compare(
|
|
3643
|
+
previous_bits, bits,
|
|
3644
|
+
plan->input_schema.fields[i].dtype) >= 0)) {
|
|
3645
|
+
code = tf_transform_set_error(
|
|
3646
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3647
|
+
"categorical plan state is inconsistent");
|
|
3648
|
+
goto done;
|
|
3649
|
+
}
|
|
3650
|
+
previous_bits = bits;
|
|
3651
|
+
if (count == SIZE_MAX) {
|
|
3652
|
+
code = tf_transform_set_error(
|
|
3653
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3654
|
+
"categorical plan width overflows");
|
|
3655
|
+
goto done;
|
|
3656
|
+
}
|
|
3657
|
+
++count;
|
|
3658
|
+
}
|
|
3659
|
+
if ((count == 0
|
|
3660
|
+
&& !(column->kind == TF_TRANSFORM_KIND_CATEGORICAL
|
|
3661
|
+
&& column->categorical_impute
|
|
3662
|
+
== TF_TRANSFORM_CATEGORICAL_IMPUTE_NONE
|
|
3663
|
+
&& column->categorical_encode
|
|
3664
|
+
== TF_TRANSFORM_ENCODE_NONE))
|
|
3665
|
+
|| (column->kind == TF_TRANSFORM_KIND_INFER
|
|
3666
|
+
&& (count < 2
|
|
3667
|
+
|| (uint64_t)count
|
|
3668
|
+
> column->infer_max_categories))) {
|
|
3669
|
+
code = tf_transform_set_error(
|
|
3670
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3671
|
+
"inferred categorical plan cardinality is inconsistent");
|
|
3672
|
+
goto done;
|
|
3673
|
+
}
|
|
3674
|
+
if ((uint64_t)count
|
|
3675
|
+
> runtime->limits.max_categories_per_column
|
|
3676
|
+
|| (uint64_t)count
|
|
3677
|
+
> runtime->limits.max_total_categories
|
|
3678
|
+
|| preflight_total_categories
|
|
3679
|
+
> runtime->limits.max_total_categories - (uint64_t)count) {
|
|
3680
|
+
code = tf_transform_set_error(
|
|
3681
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3682
|
+
"categorical plan state exceeds host limits");
|
|
3683
|
+
goto done;
|
|
3684
|
+
}
|
|
3685
|
+
preflight_total_categories += (uint64_t)count;
|
|
3686
|
+
if (column->categorical_encode == TF_TRANSFORM_ENCODE_ONEHOT) {
|
|
3687
|
+
width = (uint64_t)count;
|
|
3688
|
+
if (column->categorical_unknown
|
|
3689
|
+
== TF_TRANSFORM_UNKNOWN_OTHER) {
|
|
3690
|
+
if (width == UINT64_MAX) {
|
|
3691
|
+
code = tf_transform_set_error(
|
|
3692
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3693
|
+
"one-hot plan width overflows");
|
|
3694
|
+
goto done;
|
|
3695
|
+
}
|
|
3696
|
+
++width;
|
|
3697
|
+
}
|
|
3698
|
+
}
|
|
3699
|
+
}
|
|
3700
|
+
if (width > UINT64_MAX - expected_output_columns) {
|
|
3701
|
+
code = tf_transform_set_error(
|
|
3702
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3703
|
+
"plan output width overflows");
|
|
3704
|
+
goto done;
|
|
3705
|
+
}
|
|
3706
|
+
expected_output_columns += width;
|
|
3707
|
+
}
|
|
3708
|
+
if (expected_output_columns > plan->recipe->max_output_columns
|
|
3709
|
+
|| expected_output_columns != (uint64_t)output_column_count) {
|
|
3710
|
+
code = tf_transform_set_error(
|
|
3711
|
+
error, TF_TRANSFORM_CORRUPT_PLAN,
|
|
3712
|
+
"learned plan output width is inconsistent");
|
|
3713
|
+
goto done;
|
|
3714
|
+
}
|
|
3715
|
+
for (size_t i = 0; i < plan->recipe->column_count; ++i) {
|
|
3716
|
+
if (plan->recipe->columns[i].kind == TF_TRANSFORM_KIND_INFER
|
|
3717
|
+
&& (plan->recipe->columns[i].infer_max_categories
|
|
3718
|
+
> runtime->limits.max_categories_per_column
|
|
3719
|
+
|| plan->recipe->columns[i].infer_max_categories
|
|
3720
|
+
> runtime->limits.max_total_categories)) {
|
|
3721
|
+
code = tf_transform_set_error(
|
|
3722
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3723
|
+
"inferred category bound exceeds host limits");
|
|
3724
|
+
goto done;
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
if (expected_output_columns > runtime->limits.max_output_columns) {
|
|
3728
|
+
code = tf_transform_set_error(
|
|
3729
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT,
|
|
3730
|
+
"learned plan output width exceeds host limits");
|
|
3731
|
+
goto done;
|
|
3732
|
+
}
|
|
3733
|
+
}
|
|
3734
|
+
if (plan->input_schema.field_count > SIZE_MAX / sizeof(*plan->states)
|
|
3735
|
+
|| (uint64_t)plan->input_schema.field_count * sizeof(*plan->states)
|
|
3736
|
+
> runtime->limits.max_allocation_bytes) {
|
|
3737
|
+
code = tf_transform_set_error(
|
|
3738
|
+
error, TF_TRANSFORM_RESOURCE_LIMIT, "imported plan state exceeds limits");
|
|
3739
|
+
goto done;
|
|
3740
|
+
}
|
|
3741
|
+
plan->states = (tf_transform_column_state *)tf_transform_resource_calloc(
|
|
3742
|
+
&ledger, plan->input_schema.field_count, sizeof(*plan->states), error);
|
|
3743
|
+
if (!plan->states) {
|
|
3744
|
+
code = ledger.last_code != TF_TRANSFORM_OK
|
|
3745
|
+
? ledger.last_code : tf_transform_set_error(
|
|
3746
|
+
error, TF_TRANSFORM_ALLOCATION,
|
|
3747
|
+
"imported plan state allocation failed");
|
|
3748
|
+
goto done;
|
|
3749
|
+
}
|
|
3750
|
+
{
|
|
3751
|
+
const cJSON *step = steps->child;
|
|
3752
|
+
for (size_t i = 0; i < plan->input_schema.field_count;
|
|
3753
|
+
++i, step = step->next) {
|
|
3754
|
+
tf_transform_column_kind resolved_kind;
|
|
3755
|
+
if (i % TF_TRANSFORM_CANCEL_ITERS_V1 == 0) {
|
|
3756
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3757
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3758
|
+
}
|
|
3759
|
+
code = imported_step_kind(
|
|
3760
|
+
step, &plan->recipe->columns[i], &resolved_kind, error);
|
|
3761
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3762
|
+
plan->states[i].kind = resolved_kind;
|
|
3763
|
+
if (resolved_kind == TF_TRANSFORM_KIND_CATEGORICAL) {
|
|
3764
|
+
code = parse_categorical_step(
|
|
3765
|
+
step, &plan->recipe->columns[i],
|
|
3766
|
+
plan->input_schema.fields[i].dtype,
|
|
3767
|
+
runtime, &ledger, &total_categories,
|
|
3768
|
+
&plan->states[i].value.categorical, error);
|
|
3769
|
+
} else {
|
|
3770
|
+
code = parse_numeric_step(
|
|
3771
|
+
step, &plan->recipe->columns[i], runtime,
|
|
3772
|
+
&plan->states[i].value.numeric, error);
|
|
3773
|
+
}
|
|
3774
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3775
|
+
}
|
|
3776
|
+
}
|
|
3777
|
+
code = parse_schema_json(
|
|
3778
|
+
required_item(root, "outputSchema"), 1, runtime,
|
|
3779
|
+
&ledger, &plan->output_schema, error);
|
|
3780
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3781
|
+
code = tf_transform_output_schema_validate_contract(
|
|
3782
|
+
&plan->input_schema, plan->recipe, NULL, plan->states,
|
|
3783
|
+
&plan->output_schema, runtime, &ledger,
|
|
3784
|
+
TF_TRANSFORM_CORRUPT_PLAN, error);
|
|
3785
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3786
|
+
code = tf_transform_json_print_canonical_runtime_ledger(
|
|
3787
|
+
root, runtime, &ledger, canonical_out, canonical_len_out, error);
|
|
3788
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3789
|
+
code = tf_transform_poll_cancel(runtime, error);
|
|
3790
|
+
if (code != TF_TRANSFORM_OK) goto done;
|
|
3791
|
+
plan->import_allocation_count = ledger.allocation_count;
|
|
3792
|
+
plan->import_peak_resident_bytes = ledger.peak_resident_bytes;
|
|
3793
|
+
code = TF_TRANSFORM_OK;
|
|
3794
|
+
done:
|
|
3795
|
+
tf_transform_recipe_release(recipe);
|
|
3796
|
+
{
|
|
3797
|
+
int cleanup_cancelled = delete_json_runtime(root, runtime);
|
|
3798
|
+
tf_transform_resource_release(&ledger, ast_resident_bytes);
|
|
3799
|
+
if (code == TF_TRANSFORM_OK && cleanup_cancelled)
|
|
3800
|
+
code = tf_transform_set_error(
|
|
3801
|
+
error, TF_TRANSFORM_CANCELLED,
|
|
3802
|
+
"prepared transform cancelled during JSON cleanup");
|
|
3803
|
+
}
|
|
3804
|
+
if (code == TF_TRANSFORM_OK) {
|
|
3805
|
+
*out = plan;
|
|
3806
|
+
plan = NULL;
|
|
3807
|
+
}
|
|
3808
|
+
tf_transform_plan_release(plan);
|
|
3809
|
+
if (code != TF_TRANSFORM_OK)
|
|
3810
|
+
tf_transform_bytes_free(canonical_out, canonical_len_out);
|
|
3811
|
+
return code;
|
|
3812
|
+
}
|